mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-03 20:47:35 +03:00
Code Issues Projects Releases Packages Wiki Activity

Add new Artwork Cache Warmer

Browse source
This commit is contained in:
Deluan 2022-12-23 12:28:22 -05:00 • committed by Deluan Quintão
parent 8c1cd9c273
commit b6eb60f019
11 changed files with 501 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scanner/metadata/ffmpeg/ffmpeg.go
View file

@ -74,7 +74,7 @@ var (
// Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s
// Stream #0:0: Audio: flac, 44100 Hz, stereo, s16
audioStreamRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+.*: Audio: (.*), (.* Hz), ([\w\.]+),*(.*.,)*`)
audioStreamRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+.*: Audio: (.*), (.* Hz), ([\w.]+),*(.*.,)*`)
// Stream #0:1: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown), 600x600 [SAR 1:1 DAR 1:1], 90k tbr, 90k tbn, 90k tbc`
coverRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:.+: (Video):.*`)

52
scanner/refresher.go
View file

@ -8,6 +8,7 @@ import (
"time"
"github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
@ -20,20 +21,20 @@ import (
//
// The actual mappings happen in MediaFiles.ToAlbum() and Albums.ToAlbumArtist()
type refresher struct {
ctx context.Context
ds model.DataStore
album map[string]struct{}
artist map[string]struct{}
dirMap dirMap
ds model.DataStore
album map[string]struct{}
artist map[string]struct{}
dirMap dirMap
cacheWarmer core.ArtworkCacheWarmer
}
func newRefresher(ctx context.Context, ds model.DataStore, dirMap dirMap) *refresher {
func newRefresher(ds model.DataStore, cw core.ArtworkCacheWarmer, dirMap dirMap) *refresher {
return &refresher{
ctx: ctx,
ds: ds,
album: map[string]struct{}{},
artist: map[string]struct{}{},
dirMap: dirMap,
ds: ds,
album: map[string]struct{}{},
artist: map[string]struct{}{},
dirMap: dirMap,
cacheWarmer: cw,
}
}
@ -46,21 +47,23 @@ func (r *refresher) accumulate(mf model.MediaFile) {
}
}
func (r *refresher) flush() error {
err := r.flushMap(r.album, "album", r.refreshAlbums)
func (r *refresher) flush(ctx context.Context) error {
err := r.flushMap(ctx, r.album, "album", r.refreshAlbums)
if err != nil {
return err
}
err = r.flushMap(r.artist, "artist", r.refreshArtists)
err = r.flushMap(ctx, r.artist, "artist", r.refreshArtists)
if err != nil {
return err
}
r.album = map[string]struct{}{}
r.artist = map[string]struct{}{}
return nil
}
type refreshCallbackFunc = func(ids ...string) error
type refreshCallbackFunc = func(ctx context.Context, ids ...string) error
func (r *refresher) flushMap(m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {
func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {
if len(m) == 0 {
return nil
}
@ -71,17 +74,17 @@ func (r *refresher) flushMap(m map[string]struct{}, entity string, refresh refre
}
chunks := utils.BreakUpStringSlice(ids, 100)
for _, chunk := range chunks {
err := refresh(chunk...)
err := refresh(ctx, chunk...)
if err != nil {
log.Error(r.ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)
log.Error(ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)
return err
}
}
return nil
}
func (r *refresher) refreshAlbums(ids ...string) error {
mfs, err := r.ds.MediaFile(r.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}})
func (r *refresher) refreshAlbums(ctx context.Context, ids ...string) error {
mfs, err := r.ds.MediaFile(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}})
if err != nil {
return err
}
@ -89,7 +92,7 @@ func (r *refresher) refreshAlbums(ids ...string) error {
return nil
}
repo := r.ds.Album(r.ctx)
repo := r.ds.Album(ctx)
grouped := slice.Group(mfs, func(m model.MediaFile) string { return m.AlbumID })
for _, group := range grouped {
songs := model.MediaFiles(group)
@ -103,6 +106,7 @@ func (r *refresher) refreshAlbums(ids ...string) error {
if err != nil {
return err
}
r.cacheWarmer.PreCache(a.CoverArtID())
}
return nil
}
@ -122,8 +126,8 @@ func (r *refresher) getImageFiles(dirs []string) (string, time.Time) {
return strings.Join(imageFiles, string(filepath.ListSeparator)), updatedAt
}
func (r *refresher) refreshArtists(ids ...string) error {
albums, err := r.ds.Album(r.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_artist_id": ids}})
func (r *refresher) refreshArtists(ctx context.Context, ids ...string) error {
albums, err := r.ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_artist_id": ids}})
if err != nil {
return err
}
@ -131,7 +135,7 @@ func (r *refresher) refreshArtists(ids ...string) error {
return nil
}
repo := r.ds.Artist(r.ctx)
repo := r.ds.Artist(ctx)
grouped := slice.Group(albums, func(al model.Album) string { return al.AlbumArtistID })
for _, group := range grouped {
a := model.Albums(group).ToAlbumArtist()

30
scanner/scanner.go
View file

@ -40,12 +40,13 @@ type FolderScanner interface {
var isScanning sync.Mutex
type scanner struct {
folders map[string]FolderScanner
status map[string]*scanStatus
lock *sync.RWMutex
ds model.DataStore
pls core.Playlists
broker events.Broker
folders map[string]FolderScanner
status map[string]*scanStatus
lock *sync.RWMutex
ds model.DataStore
pls core.Playlists
broker events.Broker
cacheWarmer core.ArtworkCacheWarmer
}
type scanStatus struct {
@ -55,14 +56,15 @@ type scanStatus struct {
lastUpdate time.Time
}
func New(ds model.DataStore, playlists core.Playlists, broker events.Broker) Scanner {
func New(ds model.DataStore, playlists core.Playlists, cacheWarmer core.ArtworkCacheWarmer, broker events.Broker) Scanner {
s := &scanner{
ds: ds,
pls: playlists,
broker: broker,
folders: map[string]FolderScanner{},
status: map[string]*scanStatus{},
lock: &sync.RWMutex{},
ds: ds,
pls: playlists,
broker: broker,
folders: map[string]FolderScanner{},
status: map[string]*scanStatus{},
lock: &sync.RWMutex{},
cacheWarmer: cacheWarmer,
}
s.loadFolders()
return s
@ -242,5 +244,5 @@ func (s *scanner) loadFolders() {
}
func (s *scanner) newScanner(f model.MediaFolder) FolderScanner {
return NewTagScanner(f.Path, s.ds, s.pls)
return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer)
}

66
scanner/tag_scanner.go
View file

@ -22,19 +22,23 @@ import (
)
type TagScanner struct {
rootFolder string
ds model.DataStore
plsSync *playlistImporter
cnt *counters
mapper *mediaFileMapper
rootFolder string
ds model.DataStore
plsSync *playlistImporter
cnt *counters
mapper *mediaFileMapper
cacheWarmer core.ArtworkCacheWarmer
}
func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists) *TagScanner {
return &TagScanner{
rootFolder: rootFolder,
plsSync: newPlaylistImporter(ds, playlists, rootFolder),
ds: ds,
func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists, cacheWarmer core.ArtworkCacheWarmer) FolderScanner {
s := &TagScanner{
rootFolder: rootFolder,
plsSync: newPlaylistImporter(ds, playlists, rootFolder),
ds: ds,
cacheWarmer: cacheWarmer,
}
return s
}
type dirMap map[string]dirStats
@ -96,6 +100,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
s.cnt = &counters{}
genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx))
s.mapper = newMediaFileMapper(s.rootFolder, genres)
refresher := newRefresher(s.ds, s.cacheWarmer, allFSDirs)
foldersFound, walkerError := s.getRootFolderWalker(ctx)
for {
@ -109,7 +114,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
if s.folderHasChanged(folderStats, allDBDirs, lastModifiedSince) {
changedDirs = append(changedDirs, folderStats.Path)
log.Debug("Processing changed folder", "dir", folderStats.Path)
err := s.processChangedDir(ctx, allFSDirs, folderStats.Path, fullScan)
err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path)
if err != nil {
log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)
}
@ -128,7 +133,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog
}
for _, dir := range deletedDirs {
err := s.processDeletedDir(ctx, allFSDirs, dir)
err := s.processDeletedDir(ctx, refresher, dir)
if err != nil {
log.Error("Error removing deleted folder from DB", "dir", dir, err)
}
@ -221,9 +226,8 @@ func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs m
return deleted
}
func (s *TagScanner) processDeletedDir(ctx context.Context, allFSDirs dirMap, dir string) error {
func (s *TagScanner) processDeletedDir(ctx context.Context, refresher *refresher, dir string) error {
start := time.Now()
buffer := newRefresher(ctx, s.ds, allFSDirs)
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
if err != nil {
@ -237,17 +241,16 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, allFSDirs dirMap, di
s.cnt.deleted += c
for _, t := range mfs {
buffer.accumulate(t)
refresher.accumulate(t)
}
err = buffer.flush()
err = refresher.flush(ctx)
log.Info(ctx, "Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
return err
}
func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, dir string, fullScan bool) error {
func (s *TagScanner) processChangedDir(ctx context.Context, refresher *refresher, fullScan bool, dir string) error {
start := time.Now()
buffer := newRefresher(ctx, s.ds, allFSDirs)
// Load folder's current tracks from DB into a map
currentTracks := map[string]model.MediaFile{}
@ -296,7 +299,7 @@ func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, di
}
// Force a refresh of the album and artist, to cater for cover art files
buffer.accumulate(c)
refresher.accumulate(c)
// Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks
// are considered gone from the music folder and will be deleted from DB
@ -307,33 +310,38 @@ func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, di
numPurgedTracks := 0
if len(filesToUpdate) > 0 {
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer)
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, refresher, dir, currentTracks, filesToUpdate)
if err != nil {
return err
}
}
if len(orphanTracks) > 0 {
numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer)
numPurgedTracks, err = s.deleteOrphanSongs(ctx, refresher, dir, orphanTracks)
if err != nil {
return err
}
}
err = buffer.flush()
err = refresher.flush(ctx)
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
"deleted", numPurgedTracks, "elapsed", time.Since(start))
return err
}
func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refresher) (int, error) {
func (s *TagScanner) deleteOrphanSongs(
ctx context.Context,
refresher *refresher,
dir string,
tracksToDelete map[string]model.MediaFile,
) (int, error) {
numPurgedTracks := 0
log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
// Remaining tracks from DB that are not in the folder are deleted
for _, ct := range tracksToDelete {
numPurgedTracks++
buffer.accumulate(ct)
refresher.accumulate(ct)
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return 0, err
}
@ -342,7 +350,13 @@ func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksTo
return numPurgedTracks, nil
}
func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refresher) (int, error) {
func (s *TagScanner) addOrUpdateTracksInDB(
ctx context.Context,
refresher *refresher,
dir string,
currentTracks map[string]model.MediaFile,
filesToUpdate []string,
) (int, error) {
numUpdatedTracks := 0
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
@ -367,7 +381,7 @@ func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, curr
if err != nil {
return 0, err
}
buffer.accumulate(n)
refresher.accumulate(n)
numUpdatedTracks++
}
}