mirror of
https://github.com/navidrome/navidrome.git
synced 2025-04-03 04:27:37 +03:00
fix(server): reduce SQLite "database busy" errors (#3760)
* fix(scanner): remove transactions where they are not strictly needed Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): force setStar transaction to start as IMMEDIATE Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): encapsulated way to upgrade tx to write mode Signed-off-by: Deluan <deluan@navidrome.org> * fix(server): use tx immediate for some playlist endpoints Signed-off-by: Deluan <deluan@navidrome.org> * make more transactions immediate (#3759) --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão
GitHub
parent
d6ec52b9d4
commit
1468a56808
10 changed files with 120 additions and 115 deletions
|
|
@ -106,79 +106,75 @@ func (p *phaseMissingTracks) stages() []ppl.Stage[*missingTracks] {
|
|||
}
|
||||
|
||||
func (p *phaseMissingTracks) processMissingTracks(in *missingTracks) (*missingTracks, error) {
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
for _, ms := range in.missing {
|
||||
var exactMatch model.MediaFile
|
||||
var equivalentMatch model.MediaFile
|
||||
for _, ms := range in.missing {
|
||||
var exactMatch model.MediaFile
|
||||
var equivalentMatch model.MediaFile
|
||||
|
||||
// Identify exact and equivalent matches
|
||||
for _, mt := range in.matched {
|
||||
if ms.Equals(mt) {
|
||||
exactMatch = mt
|
||||
break // Prioritize exact match
|
||||
}
|
||||
if ms.IsEquivalent(mt) {
|
||||
equivalentMatch = mt
|
||||
}
|
||||
// Identify exact and equivalent matches
|
||||
for _, mt := range in.matched {
|
||||
if ms.Equals(mt) {
|
||||
exactMatch = mt
|
||||
break // Prioritize exact match
|
||||
}
|
||||
|
||||
// Use the exact match if found
|
||||
if exactMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track in a new place", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, exactMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error moving matched track", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// If there is only one missing and one matched track, consider them equivalent (same PID)
|
||||
if len(in.missing) == 1 && len(in.matched) == 1 {
|
||||
singleMatch := in.matched[0]
|
||||
log.Debug(p.ctx, "Scanner: Found track with same persistent ID in a new place", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, singleMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// Use the equivalent match if no other better match was found
|
||||
if equivalentMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track with same base path", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, equivalentMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
if ms.IsEquivalent(mt) {
|
||||
equivalentMatch = mt
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, "scanner: process missing tracks")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
// Use the exact match if found
|
||||
if exactMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track in a new place", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(exactMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error moving matched track", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name, err)
|
||||
return nil, err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// If there is only one missing and one matched track, consider them equivalent (same PID)
|
||||
if len(in.missing) == 1 && len(in.matched) == 1 {
|
||||
singleMatch := in.matched[0]
|
||||
log.Debug(p.ctx, "Scanner: Found track with same persistent ID in a new place", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(singleMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name, err)
|
||||
return nil, err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// Use the equivalent match if no other better match was found
|
||||
if equivalentMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track with same base path", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(equivalentMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name, err)
|
||||
return nil, err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
}
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) moveMatched(tx model.DataStore, mt, ms model.MediaFile) error {
|
||||
discardedID := mt.ID
|
||||
mt.ID = ms.ID
|
||||
err := tx.MediaFile(p.ctx).Put(&mt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update matched track: %w", err)
|
||||
}
|
||||
err = tx.MediaFile(p.ctx).Delete(discardedID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete discarded track: %w", err)
|
||||
}
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
func (p *phaseMissingTracks) moveMatched(mt, ms model.MediaFile) error {
|
||||
return p.ds.WithTx(func(tx model.DataStore) error {
|
||||
discardedID := mt.ID
|
||||
mt.ID = ms.ID
|
||||
err := tx.MediaFile(p.ctx).Put(&mt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update matched track: %w", err)
|
||||
}
|
||||
err = tx.MediaFile(p.ctx).Delete(discardedID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete discarded track: %w", err)
|
||||
}
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) finalize(err error) error {
|
||||
|
|
|
|||
|
|
@ -104,19 +104,13 @@ func (p *phaseRefreshAlbums) refreshAlbum(album *model.Album) (*model.Album, err
|
|||
return nil, nil
|
||||
}
|
||||
start := time.Now()
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
err := tx.Album(p.ctx).Put(album)
|
||||
log.Debug(p.ctx, "Scanner: refreshing album", "album_id", album.ID, "name", album.Name, "songCount", album.SongCount, "elapsed", time.Since(start))
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing album %s: %w", album.ID, err)
|
||||
}
|
||||
p.refreshed.Add(1)
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
}, "scanner: refresh album")
|
||||
err := p.ds.Album(p.ctx).Put(album)
|
||||
log.Debug(p.ctx, "Scanner: refreshing album", "album_id", album.ID, "name", album.Name, "songCount", album.SongCount, "elapsed", time.Since(start), err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("refreshing album %s: %w", album.ID, err)
|
||||
}
|
||||
p.refreshed.Add(1)
|
||||
p.state.changesDetected.Store(true)
|
||||
return album, nil
|
||||
}
|
||||
|
||||
|
|
@ -135,23 +129,21 @@ func (p *phaseRefreshAlbums) finalize(err error) error {
|
|||
log.Debug(p.ctx, "Scanner: No changes detected, skipping refreshing annotations")
|
||||
return nil
|
||||
}
|
||||
return p.ds.WithTx(func(tx model.DataStore) error {
|
||||
// Refresh album annotations
|
||||
start := time.Now()
|
||||
cnt, err := tx.Album(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing album annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed album annotations", "albums", cnt, "elapsed", time.Since(start))
|
||||
// Refresh album annotations
|
||||
start := time.Now()
|
||||
cnt, err := p.ds.Album(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing album annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed album annotations", "albums", cnt, "elapsed", time.Since(start))
|
||||
|
||||
// Refresh artist annotations
|
||||
start = time.Now()
|
||||
cnt, err = tx.Artist(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing artist annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed artist annotations", "artists", cnt, "elapsed", time.Since(start))
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
}, "scanner: finalize phaseRefreshAlbums")
|
||||
// Refresh artist annotations
|
||||
start = time.Now()
|
||||
cnt, err = p.ds.Artist(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing artist annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed artist annotations", "artists", cnt, "elapsed", time.Since(start))
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -138,24 +138,22 @@ func (s *scannerImpl) runRefreshStats(ctx context.Context, state *scanState) fun
|
|||
log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats")
|
||||
return nil
|
||||
}
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
start := time.Now()
|
||||
stats, err := tx.Artist(ctx).RefreshStats()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error refreshing artists stats", err)
|
||||
return fmt.Errorf("refreshing artists stats: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
|
||||
start := time.Now()
|
||||
stats, err := s.ds.Artist(ctx).RefreshStats()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error refreshing artists stats", err)
|
||||
return fmt.Errorf("refreshing artists stats: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
|
||||
|
||||
start = time.Now()
|
||||
err = tx.Tag(ctx).UpdateCounts()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating tag counts", err)
|
||||
return fmt.Errorf("updating tag counts: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start))
|
||||
return nil
|
||||
}, "scanner: refresh stats")
|
||||
start = time.Now()
|
||||
err = s.ds.Tag(ctx).UpdateCounts()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating tag counts", err)
|
||||
return fmt.Errorf("updating tag counts: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue