From 1a96e9fe65ffa3bc0bc6a8fb06d2ffefa3c31e0d Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 18 Oct 2021 22:17:29 -0400 Subject: [PATCH] Import smart playlists (extension .nsp) --- cmd/wire_gen.go | 6 +- scanner/playlist_sync.go => core/playlists.go | 111 ++++++++++-------- core/playlists_test.go | 90 ++++++++++++++ core/wire_providers.go | 1 + scanner/playlist_importer.go | 63 ++++++++++ ...sync_test.go => playlist_importer_test.go} | 44 ++----- scanner/scanner.go | 6 +- scanner/tag_scanner.go | 6 +- scanner/walk_dir_tree.go | 3 +- server/subsonic/wire_gen.go | 3 +- utils/files.go | 5 - utils/files_test.go | 14 --- 12 files changed, 242 insertions(+), 110 deletions(-) rename scanner/playlist_sync.go => core/playlists.go (56%) create mode 100644 core/playlists_test.go create mode 100644 scanner/playlist_importer.go rename scanner/{playlist_sync_test.go => playlist_importer_test.go} (62%) diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index f28285878..0ec680c50 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire -//+build !wireinject +//go:build !wireinject +// +build !wireinject package cmd @@ -69,11 +70,12 @@ func CreateLastFMRouter() *lastfm.Router { func createScanner() scanner.Scanner { sqlDB := db.Db() dataStore := persistence.New(sqlDB) + playlists := core.NewPlaylists(dataStore) artworkCache := core.GetImageCache() artwork := core.NewArtwork(dataStore, artworkCache) cacheWarmer := core.NewCacheWarmer(artwork, artworkCache) broker := events.GetBroker() - scannerScanner := scanner.New(dataStore, cacheWarmer, broker) + scannerScanner := scanner.New(dataStore, playlists, cacheWarmer, broker) return scannerScanner } diff --git a/scanner/playlist_sync.go b/core/playlists.go similarity index 56% rename from scanner/playlist_sync.go rename to core/playlists.go index d469e96ce..f795be248 100644 --- a/scanner/playlist_sync.go +++ b/core/playlists.go @@ -1,68 +1,76 @@ -package scanner +package core import ( "bufio" "bytes" "context" + "encoding/json" "fmt" + "io" "net/url" "os" "path/filepath" "strings" - "github.com/mattn/go-zglob" - "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" - "github.com/navidrome/navidrome/utils" ) -type playlistSync struct { - ds model.DataStore - rootFolder string +type Playlists interface { + ImportFile(ctx context.Context, dir string, fname string) (*model.Playlist, error) } -func newPlaylistSync(ds model.DataStore, rootFolder string) *playlistSync { - return &playlistSync{ds: ds, rootFolder: rootFolder} +type playlists struct { + ds model.DataStore } -func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int64 { - if !s.inPlaylistsPath(dir) { - return 0 - } - var count int64 - files, err := os.ReadDir(dir) +func NewPlaylists(ds model.DataStore) Playlists { + return &playlists{ds: ds} +} + +func IsPlaylist(filePath string) bool { + extension := strings.ToLower(filepath.Ext(filePath)) + return extension == ".m3u" || extension == ".m3u8" || extension == ".nsp" +} + +func (s *playlists) ImportFile(ctx context.Context, dir string, fname string) (*model.Playlist, error) { + pls, err := s.parsePlaylist(ctx, fname, dir) if err != nil { - log.Error(ctx, "Error reading files", "dir", dir, err) - return count + log.Error(ctx, "Error parsing playlist", "playlist", fname, err) + return nil, err } - for _, f := range files { - if !utils.IsPlaylist(f.Name()) { - continue - } - pls, err := s.parsePlaylist(ctx, f.Name(), dir) - if err != nil { - log.Error(ctx, "Error parsing playlist", "playlist", f.Name(), err) - continue - } - log.Debug("Found playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks)) - err = s.updatePlaylist(ctx, pls) - if err != nil { - log.Error(ctx, "Error updating playlist", "playlist", f.Name(), err) - } - count++ + log.Debug("Found playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks)) + err = s.updatePlaylist(ctx, pls) + if err != nil { + log.Error(ctx, "Error updating playlist", "playlist", fname, err) } - return count + return pls, err } -func (s *playlistSync) parsePlaylist(ctx context.Context, playlistFile string, baseDir string) (*model.Playlist, error) { - playlistPath := filepath.Join(baseDir, playlistFile) - file, err := os.Open(playlistPath) +func (s *playlists) parsePlaylist(ctx context.Context, playlistFile string, baseDir string) (*model.Playlist, error) { + pls, err := s.newSyncedPlaylist(baseDir, playlistFile) + if err != nil { + return nil, err + } + + file, err := os.Open(pls.Path) if err != nil { return nil, err } defer file.Close() + + extension := strings.ToLower(filepath.Ext(playlistFile)) + switch extension { + case ".nsp": + return s.parseNSP(ctx, pls, file) + default: + return s.parseM3U(ctx, pls, baseDir, file) + } +} + +func (s *playlists) newSyncedPlaylist(baseDir string, playlistFile string) (*model.Playlist, error) { + playlistPath := filepath.Join(baseDir, playlistFile) info, err := os.Stat(playlistPath) if err != nil { return nil, err @@ -79,7 +87,24 @@ func (s *playlistSync) parsePlaylist(ctx context.Context, playlistFile string, b Sync: true, UpdatedAt: info.ModTime(), } + return pls, nil +} +func (s *playlists) parseNSP(ctx context.Context, pls *model.Playlist, file io.Reader) (*model.Playlist, error) { + content, err := io.ReadAll(file) + if err != nil { + return nil, err + } + + pls.Rules = &model.SmartPlaylist{} + err = json.Unmarshal(content, pls.Rules) + if err != nil { + return nil, err + } + return pls, nil +} + +func (s *playlists) parseM3U(ctx context.Context, pls *model.Playlist, baseDir string, file io.Reader) (*model.Playlist, error) { mediaFileRepository := s.ds.MediaFile(ctx) scanner := bufio.NewScanner(file) scanner.Split(scanLines) @@ -99,7 +124,7 @@ func (s *playlistSync) parsePlaylist(ctx context.Context, playlistFile string, b } mf, err := mediaFileRepository.FindByPath(path) if err != nil { - log.Warn(ctx, "Path in playlist not found", "playlist", playlistFile, "path", path, err) + log.Warn(ctx, "Path in playlist not found", "playlist", pls.Name, "path", path, err) continue } mfs = append(mfs, *mf) @@ -110,7 +135,7 @@ func (s *playlistSync) parsePlaylist(ctx context.Context, playlistFile string, b return pls, scanner.Err() } -func (s *playlistSync) updatePlaylist(ctx context.Context, newPls *model.Playlist) error { +func (s *playlists) updatePlaylist(ctx context.Context, newPls *model.Playlist) error { owner, _ := request.UsernameFrom(ctx) pls, err := s.ds.Playlist(ctx).FindByPath(newPls.Path) @@ -136,16 +161,6 @@ func (s *playlistSync) updatePlaylist(ctx context.Context, newPls *model.Playlis return s.ds.Playlist(ctx).Put(newPls) } -func (s *playlistSync) inPlaylistsPath(dir string) bool { - rel, _ := filepath.Rel(s.rootFolder, dir) - for _, path := range strings.Split(conf.Server.PlaylistsPath, string(filepath.ListSeparator)) { - if match, _ := zglob.Match(path, rel); match { - return true - } - } - return false -} - // From https://stackoverflow.com/a/41433698 func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { diff --git a/core/playlists_test.go b/core/playlists_test.go new file mode 100644 index 000000000..9eb5c3388 --- /dev/null +++ b/core/playlists_test.go @@ -0,0 +1,90 @@ +package core + +import ( + "context" + "path/filepath" + + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("IsPlaylist", func() { + It("returns true for a M3U file", func() { + Expect(IsPlaylist(filepath.Join("path", "to", "test.m3u"))).To(BeTrue()) + }) + + It("returns true for a M3U8 file", func() { + Expect(IsPlaylist(filepath.Join("path", "to", "test.m3u8"))).To(BeTrue()) + }) + + It("returns false for a non-playlist file", func() { + Expect(IsPlaylist("testm3u")).To(BeFalse()) + }) +}) + +var _ = Describe("Playlists", func() { + var ds model.DataStore + var ps Playlists + ctx := context.Background() + + BeforeEach(func() { + ds = &tests.MockDataStore{ + MockedMediaFile: &mockedMediaFile{}, + MockedPlaylist: &mockedPlaylist{}, + } + }) + + Describe("ImportFile", func() { + BeforeEach(func() { + ps = NewPlaylists(ds) + }) + + It("parses well-formed playlists", func() { + pls, err := ps.ImportFile(ctx, "tests/fixtures", "playlists/pls1.m3u") + Expect(err).To(BeNil()) + Expect(pls.Tracks).To(HaveLen(3)) + Expect(pls.Tracks[0].Path).To(Equal("tests/fixtures/test.mp3")) + Expect(pls.Tracks[1].Path).To(Equal("tests/fixtures/test.ogg")) + Expect(pls.Tracks[2].Path).To(Equal("/tests/fixtures/01 Invisible (RED) Edit Version.mp3")) + }) + + It("parses playlists using LF ending", func() { + pls, err := ps.ImportFile(ctx, "tests/fixtures/playlists", "lf-ended.m3u") + Expect(err).To(BeNil()) + Expect(pls.Tracks).To(HaveLen(2)) + }) + + It("parses playlists using CR ending (old Mac format)", func() { + pls, err := ps.ImportFile(ctx, "tests/fixtures/playlists", "cr-ended.m3u") + Expect(err).To(BeNil()) + Expect(pls.Tracks).To(HaveLen(2)) + }) + + }) +}) + +type mockedMediaFile struct { + model.MediaFileRepository +} + +func (r *mockedMediaFile) FindByPath(s string) (*model.MediaFile, error) { + return &model.MediaFile{ + ID: "123", + Path: s, + }, nil +} + +type mockedPlaylist struct { + model.PlaylistRepository +} + +func (r *mockedPlaylist) FindByPath(path string) (*model.Playlist, error) { + return nil, model.ErrNotFound +} + +func (r *mockedPlaylist) Put(pls *model.Playlist) error { + return nil +} diff --git a/core/wire_providers.go b/core/wire_providers.go index b5b60cf8f..37cb7efdd 100644 --- a/core/wire_providers.go +++ b/core/wire_providers.go @@ -20,4 +20,5 @@ var Set = wire.NewSet( transcoder.New, scrobbler.GetPlayTracker, NewShare, + NewPlaylists, ) diff --git a/scanner/playlist_importer.go b/scanner/playlist_importer.go new file mode 100644 index 000000000..c6f9f4b4a --- /dev/null +++ b/scanner/playlist_importer.go @@ -0,0 +1,63 @@ +package scanner + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/mattn/go-zglob" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/core" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +type playlistImporter struct { + ds model.DataStore + pls core.Playlists + rootFolder string +} + +func newPlaylistImporter(ds model.DataStore, playlists core.Playlists, rootFolder string) *playlistImporter { + return &playlistImporter{ds: ds, pls: playlists, rootFolder: rootFolder} +} + +func (s *playlistImporter) processPlaylists(ctx context.Context, dir string) int64 { + if !s.inPlaylistsPath(dir) { + return 0 + } + var count int64 + files, err := os.ReadDir(dir) + if err != nil { + log.Error(ctx, "Error reading files", "dir", dir, err) + return count + } + for _, f := range files { + if !core.IsPlaylist(f.Name()) { + continue + } + pls, err := s.pls.ImportFile(ctx, dir, f.Name()) + if err != nil { + log.Error(ctx, "Error parsing playlist", "playlist", f.Name(), err) + continue + } + if pls.IsSmartPlaylist() { + log.Debug("Imported smart playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks)) + } else { + log.Debug("Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks)) + } + count++ + } + return count +} + +func (s *playlistImporter) inPlaylistsPath(dir string) bool { + rel, _ := filepath.Rel(s.rootFolder, dir) + for _, path := range strings.Split(conf.Server.PlaylistsPath, string(filepath.ListSeparator)) { + if match, _ := zglob.Match(path, rel); match { + return true + } + } + return false +} diff --git a/scanner/playlist_sync_test.go b/scanner/playlist_importer_test.go similarity index 62% rename from scanner/playlist_sync_test.go rename to scanner/playlist_importer_test.go index d3f95889e..6ae7c4f0c 100644 --- a/scanner/playlist_sync_test.go +++ b/scanner/playlist_importer_test.go @@ -3,6 +3,8 @@ package scanner import ( "context" + "github.com/navidrome/navidrome/core" + "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/model" @@ -11,9 +13,10 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("playlistSync", func() { +var _ = Describe("playlistImporter", func() { var ds model.DataStore - var ps *playlistSync + var ps *playlistImporter + var pls core.Playlists ctx := context.Background() BeforeEach(func() { @@ -21,34 +24,7 @@ var _ = Describe("playlistSync", func() { MockedMediaFile: &mockedMediaFile{}, MockedPlaylist: &mockedPlaylist{}, } - }) - - Describe("parsePlaylist", func() { - BeforeEach(func() { - ps = newPlaylistSync(ds, "tests/") - }) - - It("parses well-formed playlists", func() { - pls, err := ps.parsePlaylist(ctx, "playlists/pls1.m3u", "tests/fixtures") - Expect(err).To(BeNil()) - Expect(pls.Tracks).To(HaveLen(3)) - Expect(pls.Tracks[0].Path).To(Equal("tests/fixtures/test.mp3")) - Expect(pls.Tracks[1].Path).To(Equal("tests/fixtures/test.ogg")) - Expect(pls.Tracks[2].Path).To(Equal("/tests/fixtures/01 Invisible (RED) Edit Version.mp3")) - }) - - It("parses playlists using LF ending", func() { - pls, err := ps.parsePlaylist(ctx, "lf-ended.m3u", "tests/fixtures/playlists") - Expect(err).To(BeNil()) - Expect(pls.Tracks).To(HaveLen(2)) - }) - - It("parses playlists using CR ending (old Mac format)", func() { - pls, err := ps.parsePlaylist(ctx, "cr-ended.m3u", "tests/fixtures/playlists") - Expect(err).To(BeNil()) - Expect(pls.Tracks).To(HaveLen(2)) - }) - + pls = core.NewPlaylists(ds) }) Describe("processPlaylists", func() { @@ -57,19 +33,19 @@ var _ = Describe("playlistSync", func() { conf.Server.PlaylistsPath = consts.DefaultPlaylistsPath }) It("finds and import playlists at the top level", func() { - ps = newPlaylistSync(ds, "tests/fixtures/playlists/subfolder1") + ps = newPlaylistImporter(ds, pls, "tests/fixtures/playlists/subfolder1") Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1))) }) It("finds and import playlists at any subfolder level", func() { - ps = newPlaylistSync(ds, "tests") + ps = newPlaylistImporter(ds, pls, "tests") Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1))) }) }) It("ignores playlists not in the PlaylistsPath", func() { conf.Server.PlaylistsPath = "subfolder1" - ps = newPlaylistSync(ds, "tests/fixtures/playlists") + ps = newPlaylistImporter(ds, pls, "tests/fixtures/playlists") Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1))) Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder2")).To(Equal(int64(0))) @@ -77,7 +53,7 @@ var _ = Describe("playlistSync", func() { It("only imports playlists from the root of MusicFolder if PlaylistsPath is '.'", func() { conf.Server.PlaylistsPath = "." - ps = newPlaylistSync(ds, "tests/fixtures/playlists") + ps = newPlaylistImporter(ds, pls, "tests/fixtures/playlists") Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists")).To(Equal(int64(3))) Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(0))) diff --git a/scanner/scanner.go b/scanner/scanner.go index 1d15359bd..58712be98 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -46,6 +46,7 @@ type scanner struct { status map[string]*scanStatus lock *sync.RWMutex ds model.DataStore + pls core.Playlists cacheWarmer core.CacheWarmer broker events.Broker } @@ -57,9 +58,10 @@ type scanStatus struct { lastUpdate time.Time } -func New(ds model.DataStore, cacheWarmer core.CacheWarmer, broker events.Broker) Scanner { +func New(ds model.DataStore, playlists core.Playlists, cacheWarmer core.CacheWarmer, broker events.Broker) Scanner { s := &scanner{ ds: ds, + pls: playlists, cacheWarmer: cacheWarmer, broker: broker, folders: map[string]FolderScanner{}, @@ -250,5 +252,5 @@ func (s *scanner) loadFolders() { } func (s *scanner) newScanner(f model.MediaFolder) FolderScanner { - return NewTagScanner(f.Path, s.ds, s.cacheWarmer) + return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer) } diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index c841b9f93..0e0ed7f8e 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -22,15 +22,15 @@ type TagScanner struct { rootFolder string ds model.DataStore cacheWarmer core.CacheWarmer - plsSync *playlistSync + plsSync *playlistImporter cnt *counters mapper *mediaFileMapper } -func NewTagScanner(rootFolder string, ds model.DataStore, cacheWarmer core.CacheWarmer) *TagScanner { +func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists, cacheWarmer core.CacheWarmer) *TagScanner { return &TagScanner{ rootFolder: rootFolder, - plsSync: newPlaylistSync(ds, rootFolder), + plsSync: newPlaylistImporter(ds, playlists, rootFolder), ds: ds, cacheWarmer: cacheWarmer, } diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 77db41828..5d3884b87 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -10,6 +10,7 @@ import ( "time" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/utils" ) @@ -95,7 +96,7 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) { if utils.IsAudioFile(entry.Name()) { stats.AudioFilesCount++ } else { - stats.HasPlaylist = stats.HasPlaylist || utils.IsPlaylist(entry.Name()) + stats.HasPlaylist = stats.HasPlaylist || core.IsPlaylist(entry.Name()) stats.HasImages = stats.HasImages || utils.IsImageFile(entry.Name()) } } diff --git a/server/subsonic/wire_gen.go b/server/subsonic/wire_gen.go index 8d76add06..7f0ea2593 100644 --- a/server/subsonic/wire_gen.go +++ b/server/subsonic/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire -//+build !wireinject +//go:build !wireinject +// +build !wireinject package subsonic diff --git a/utils/files.go b/utils/files.go index a3a071484..5e9150c94 100644 --- a/utils/files.go +++ b/utils/files.go @@ -21,8 +21,3 @@ func IsImageFile(filePath string) bool { extension := filepath.Ext(filePath) return strings.HasPrefix(mime.TypeByExtension(extension), "image/") } - -func IsPlaylist(filePath string) bool { - extension := strings.ToLower(filepath.Ext(filePath)) - return extension == ".m3u" || extension == ".m3u8" -} diff --git a/utils/files_test.go b/utils/files_test.go index 97f54b7a8..cc237ce5b 100644 --- a/utils/files_test.go +++ b/utils/files_test.go @@ -43,18 +43,4 @@ var _ = Describe("Files", func() { Expect(IsImageFile("test.mp3")).To(BeFalse()) }) }) - - Describe("IsPlaylist", func() { - It("returns true for a M3U file", func() { - Expect(IsPlaylist(filepath.Join("path", "to", "test.m3u"))).To(BeTrue()) - }) - - It("returns true for a M3U8 file", func() { - Expect(IsPlaylist(filepath.Join("path", "to", "test.m3u8"))).To(BeTrue()) - }) - - It("returns false for a non-playlist file", func() { - Expect(IsPlaylist("testm3u")).To(BeFalse()) - }) - }) })