mirror of
https://github.com/navidrome/navidrome.git
synced 2025-04-04 04:57:37 +03:00
WIP: Start scanner2, using charlievieth/fastwalk package
This commit is contained in:
Deluan
parent
eec0d880c3
commit
b2dcc0b900
7 changed files with 114 additions and 38 deletions
14
cmd/root.go
14
cmd/root.go
|
|
@ -97,15 +97,15 @@ func runNavidrome() {
|
|||
|
||||
func startServer(ctx context.Context) func() error {
|
||||
return func() error {
|
||||
a := CreateServer()
|
||||
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter())
|
||||
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter())
|
||||
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter())
|
||||
a := CreateServer(ctx)
|
||||
a.MountRouter("Native API", consts.URLPathNativeAPI, CreateNativeAPIRouter(ctx))
|
||||
a.MountRouter("Subsonic API", consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter(ctx))
|
||||
a.MountRouter("Public Endpoints", consts.URLPathPublic, CreatePublicRouter(ctx))
|
||||
if conf.Server.LastFM.Enabled {
|
||||
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter())
|
||||
a.MountRouter("LastFM Auth", consts.URLPathNativeAPI+"/lastfm", CreateLastFMRouter(ctx))
|
||||
}
|
||||
if conf.Server.ListenBrainz.Enabled {
|
||||
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter())
|
||||
a.MountRouter("ListenBrainz Auth", consts.URLPathNativeAPI+"/listenbrainz", CreateListenBrainzRouter(ctx))
|
||||
}
|
||||
if conf.Server.Prometheus.Enabled {
|
||||
// blocking call because takes <1ms but useful if fails
|
||||
|
|
@ -130,7 +130,7 @@ func schedulePeriodicScan(ctx context.Context) func() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
scanner := GetScanner()
|
||||
scanner := GetScanner(ctx)
|
||||
schedulerInstance := scheduler.GetInstance()
|
||||
|
||||
log.Info("Scheduling periodic scan", "schedule", schedule)
|
||||
|
|
|
|||
|
|
@ -24,8 +24,9 @@ var scanCmd = &cobra.Command{
|
|||
}
|
||||
|
||||
func runScanner() {
|
||||
scanner := GetScanner()
|
||||
_ = scanner.RescanAll(context.Background(), fullRescan)
|
||||
ctx := context.Background()
|
||||
scanner := GetScanner(ctx)
|
||||
_ = scanner.RescanAll(ctx, fullRescan)
|
||||
if fullRescan {
|
||||
log.Info("Finished full rescan")
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/google/wire"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/agents"
|
||||
|
|
@ -18,6 +19,7 @@ import (
|
|||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/scanner2"
|
||||
"github.com/navidrome/navidrome/server"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/server/nativeapi"
|
||||
|
|
@ -28,7 +30,7 @@ import (
|
|||
|
||||
// Injectors from wire_injectors.go:
|
||||
|
||||
func CreateServer() *server.Server {
|
||||
func CreateServer(ctx context.Context) *server.Server {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
broker := events.GetBroker()
|
||||
|
|
@ -36,7 +38,7 @@ func CreateServer() *server.Server {
|
|||
return serverServer
|
||||
}
|
||||
|
||||
func CreateNativeAPIRouter() *nativeapi.Router {
|
||||
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
share := core.NewShare(dataStore)
|
||||
|
|
@ -45,7 +47,7 @@ func CreateNativeAPIRouter() *nativeapi.Router {
|
|||
return router
|
||||
}
|
||||
|
||||
func CreateSubsonicAPIRouter() *subsonic.Router {
|
||||
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
fileCache := artwork.GetImageCache()
|
||||
|
|
@ -58,7 +60,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
|
|||
share := core.NewShare(dataStore)
|
||||
archiver := core.NewArchiver(mediaStreamer, dataStore, share)
|
||||
players := core.NewPlayers(dataStore)
|
||||
scanner := GetScanner()
|
||||
scanner := GetScanner(ctx)
|
||||
broker := events.GetBroker()
|
||||
playlists := core.NewPlaylists(dataStore)
|
||||
playTracker := scrobbler.GetPlayTracker(dataStore, broker)
|
||||
|
|
@ -66,7 +68,7 @@ func CreateSubsonicAPIRouter() *subsonic.Router {
|
|||
return router
|
||||
}
|
||||
|
||||
func CreatePublicRouter() *public.Router {
|
||||
func CreatePublicRouter(ctx context.Context) *public.Router {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
fileCache := artwork.GetImageCache()
|
||||
|
|
@ -82,32 +84,24 @@ func CreatePublicRouter() *public.Router {
|
|||
return router
|
||||
}
|
||||
|
||||
func CreateLastFMRouter() *lastfm.Router {
|
||||
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
router := lastfm.NewRouter(dataStore)
|
||||
return router
|
||||
}
|
||||
|
||||
func CreateListenBrainzRouter() *listenbrainz.Router {
|
||||
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
router := listenbrainz.NewRouter(dataStore)
|
||||
return router
|
||||
}
|
||||
|
||||
func createScanner() scanner.Scanner {
|
||||
func createScanner(ctx context.Context) scanner.Scanner {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
playlists := core.NewPlaylists(dataStore)
|
||||
fileCache := artwork.GetImageCache()
|
||||
fFmpeg := ffmpeg.New()
|
||||
agentsAgents := agents.New(dataStore)
|
||||
externalMetadata := core.NewExternalMetadata(dataStore, agentsAgents)
|
||||
artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, externalMetadata)
|
||||
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
|
||||
broker := events.GetBroker()
|
||||
scannerScanner := scanner.New(dataStore, playlists, cacheWarmer, broker)
|
||||
scannerScanner := scanner2.New(ctx, dataStore)
|
||||
return scannerScanner
|
||||
}
|
||||
|
||||
|
|
@ -121,9 +115,9 @@ var (
|
|||
scannerInstance scanner.Scanner
|
||||
)
|
||||
|
||||
func GetScanner() scanner.Scanner {
|
||||
func GetScanner(ctx context.Context) scanner.Scanner {
|
||||
onceScanner.Do(func() {
|
||||
scannerInstance = createScanner()
|
||||
scannerInstance = createScanner(ctx)
|
||||
})
|
||||
return scannerInstance
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/google/wire"
|
||||
|
|
@ -13,6 +14,7 @@ import (
|
|||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/scanner2"
|
||||
"github.com/navidrome/navidrome/server"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/server/nativeapi"
|
||||
|
|
@ -33,39 +35,39 @@ var allProviders = wire.NewSet(
|
|||
db.Db,
|
||||
)
|
||||
|
||||
func CreateServer() *server.Server {
|
||||
func CreateServer(ctx context.Context) *server.Server {
|
||||
panic(wire.Build(
|
||||
server.New,
|
||||
allProviders,
|
||||
))
|
||||
}
|
||||
|
||||
func CreateNativeAPIRouter() *nativeapi.Router {
|
||||
func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
}
|
||||
|
||||
func CreateSubsonicAPIRouter() *subsonic.Router {
|
||||
func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
GetScanner,
|
||||
))
|
||||
}
|
||||
|
||||
func CreatePublicRouter() *public.Router {
|
||||
func CreatePublicRouter(ctx context.Context) *public.Router {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
}
|
||||
|
||||
func CreateLastFMRouter() *lastfm.Router {
|
||||
func CreateLastFMRouter(ctx context.Context) *lastfm.Router {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
}
|
||||
|
||||
func CreateListenBrainzRouter() *listenbrainz.Router {
|
||||
func CreateListenBrainzRouter(ctx context.Context) *listenbrainz.Router {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
|
|
@ -77,16 +79,16 @@ var (
|
|||
scannerInstance scanner.Scanner
|
||||
)
|
||||
|
||||
func GetScanner() scanner.Scanner {
|
||||
func GetScanner(ctx context.Context) scanner.Scanner {
|
||||
onceScanner.Do(func() {
|
||||
scannerInstance = createScanner()
|
||||
scannerInstance = createScanner(ctx)
|
||||
})
|
||||
return scannerInstance
|
||||
}
|
||||
|
||||
func createScanner() scanner.Scanner {
|
||||
func createScanner(ctx context.Context) scanner.Scanner {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
scanner.New,
|
||||
scanner2.New,
|
||||
))
|
||||
}
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -6,6 +6,7 @@ require (
|
|||
github.com/Masterminds/squirrel v1.5.4
|
||||
github.com/RaveNoX/go-jsoncommentstrip v1.0.0
|
||||
github.com/bradleyjkemp/cupaloy/v2 v2.8.0
|
||||
github.com/charlievieth/fastwalk v1.0.1
|
||||
github.com/deluan/rest v0.0.0-20211102003136-6260bc399cbf
|
||||
github.com/deluan/sanitize v0.0.0-20230310221930-6e18967d9fc1
|
||||
github.com/dexterlb/mpvipc v0.0.0-20230829142118-145d6eabdc37
|
||||
|
|
|
|||
1
go.sum
1
go.sum
|
|
@ -12,6 +12,7 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM
|
|||
github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charlievieth/fastwalk v1.0.1/go.mod h1:dryXgMJyGHbMrAmmnF0/EJNBbZaihlwcNud5IuGyogU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
|
|||
77
scanner2/scanner2.go
Normal file
77
scanner2/scanner2.go
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
package scanner2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
|
||||
"github.com/charlievieth/fastwalk"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
)
|
||||
|
||||
type scanner2 struct {
|
||||
ctx context.Context
|
||||
ds model.DataStore
|
||||
}
|
||||
|
||||
func New(ctx context.Context, ds model.DataStore) scanner.Scanner {
|
||||
return &scanner2{ctx: ctx, ds: ds}
|
||||
}
|
||||
|
||||
func (s *scanner2) RescanAll(ctx context.Context, fullRescan bool) error {
|
||||
libs, err := s.ds.Library(s.ctx).GetAll()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
libsChan := pl.FromSlice(s.ctx, libs)
|
||||
folderChan, folderErrChan := walkDirEntries(s.ctx, libsChan)
|
||||
for folder := range folderChan {
|
||||
log.Debug(s.ctx, "Scanner: Found folder", "folder", folder.Name())
|
||||
}
|
||||
|
||||
// Wait for pipeline to end, return first error found
|
||||
for err := range pl.Merge(ctx, folderErrChan) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func walkDirEntries(ctx context.Context, libsChan <-chan model.Library) (chan fastwalk.DirEntry, chan error) {
|
||||
outputChannel := make(chan fastwalk.DirEntry)
|
||||
errChannel := make(chan error)
|
||||
go func() {
|
||||
defer close(outputChannel)
|
||||
defer close(errChannel)
|
||||
errChan := pl.Sink(ctx, 1, libsChan, func(ctx context.Context, lib model.Library) error {
|
||||
conf := &fastwalk.Config{Follow: true}
|
||||
return fastwalk.Walk(conf, lib.Path, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error walking path", "lib", lib.Name, "path", path, err)
|
||||
}
|
||||
if d.IsDir() {
|
||||
outputChannel <- d.(fastwalk.DirEntry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
// Wait for pipeline to end, and forward any errors
|
||||
for err := range pl.ReadOrDone(ctx, errChan) {
|
||||
errChannel <- err
|
||||
}
|
||||
}()
|
||||
return outputChannel, errChannel
|
||||
}
|
||||
func (s *scanner2) Status(context.Context) (*scanner.StatusInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
//nolint:unused
|
||||
func (s *scanner2) doScan(ctx context.Context, lib model.Library, fullRescan bool, folders <-chan string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ scanner.Scanner = (*scanner2)(nil)
|
||||
Loading…
Add table
Add a link
Reference in a new issue