mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-03 20:47:35 +03:00
Code Issues Projects Releases Packages Wiki Activity

fix(server): reduce SQLite "database busy" errors (#3760)

Browse source 
* fix(scanner): remove transactions where they are not strictly needed

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(server): force setStar transaction to start as IMMEDIATE

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(server): encapsulated way to upgrade tx to write mode

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(server): use tx immediate for some playlist endpoints

Signed-off-by: Deluan <deluan@navidrome.org>

* make more transactions immediate (#3759)

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão 2025-02-26 19:01:49 -08:00 • committed by GitHub
parent d6ec52b9d4
commit 1468a56808
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 120 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/playlists.go
View file

@ -262,7 +262,7 @@ func (s *playlists) Update(ctx context.Context, playlistID string,
needsInfoUpdate := name != nil || comment != nil || public != nil needsInfoUpdate := name != nil || comment != nil || public != nil
needsTrackRefresh := len(idxToRemove) > 0 needsTrackRefresh := len(idxToRemove) > 0
return s.ds.WithTx(func(tx model.DataStore) error { return s.ds.WithTxImmediate(func(tx model.DataStore) error {
var pls *model.Playlist var pls *model.Playlist
var err error var err error
repo := tx.Playlist(ctx) repo := tx.Playlist(ctx)

1
model/datastore.go
View file

@ -42,5 +42,6 @@ type DataStore interface {
Resource(ctx context.Context, model interface{}) ResourceRepository Resource(ctx context.Context, model interface{}) ResourceRepository
WithTx(block func(tx DataStore) error, scope ...string) error WithTx(block func(tx DataStore) error, scope ...string) error
WithTxImmediate(block func(tx DataStore) error, scope ...string) error
GC(ctx context.Context) error GC(ctx context.Context) error
} }

14
persistence/persistence.go
View file

@ -143,6 +143,20 @@ func (s *SQLStore) WithTx(block func(tx model.DataStore) error, scope ...string)
}) })
} }
func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope ...string) error {
ctx := context.Background()
return s.WithTx(func(tx model.DataStore) error {
// Workaround to force the transaction to be upgraded to immediate mode to avoid deadlocks
// See https://berthub.eu/articles/posts/a-brief-post-on-sqlite3-database-locked-despite-timeout/
_ = tx.Property(ctx).Put("tmp_lock_flag", "")
defer func() {
_ = tx.Property(ctx).Delete("tmp_lock_flag")
}()
return block(tx)
}, scope...)
}
func (s *SQLStore) GC(ctx context.Context) error { func (s *SQLStore) GC(ctx context.Context) error {
trace := func(ctx context.Context, msg string, f func() error) func() error { trace := func(ctx context.Context, msg string, f func() error) func() error {
return func() error { return func() error {

126
scanner/phase_2_missing_tracks.go
View file

@ -106,79 +106,75 @@ func (p *phaseMissingTracks) stages() []ppl.Stage[*missingTracks] {
} }
func (p *phaseMissingTracks) processMissingTracks(in *missingTracks) (*missingTracks, error) { func (p *phaseMissingTracks) processMissingTracks(in *missingTracks) (*missingTracks, error) {
err := p.ds.WithTx(func(tx model.DataStore) error { for _, ms := range in.missing {
for _, ms := range in.missing { var exactMatch model.MediaFile
var exactMatch model.MediaFile var equivalentMatch model.MediaFile
var equivalentMatch model.MediaFile
// Identify exact and equivalent matches // Identify exact and equivalent matches
for _, mt := range in.matched { for _, mt := range in.matched {
if ms.Equals(mt) { if ms.Equals(mt) {
exactMatch = mt exactMatch = mt
break // Prioritize exact match break // Prioritize exact match
}
if ms.IsEquivalent(mt) {
equivalentMatch = mt
}
} }
if ms.IsEquivalent(mt) {
// Use the exact match if found equivalentMatch = mt
if exactMatch.ID != "" {
log.Debug(p.ctx, "Scanner: Found missing track in a new place", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(tx, exactMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error moving matched track", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name, err)
return err
}
p.totalMatched.Add(1)
continue
}
// If there is only one missing and one matched track, consider them equivalent (same PID)
if len(in.missing) == 1 && len(in.matched) == 1 {
singleMatch := in.matched[0]
log.Debug(p.ctx, "Scanner: Found track with same persistent ID in a new place", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(tx, singleMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name, err)
return err
}
p.totalMatched.Add(1)
continue
}
// Use the equivalent match if no other better match was found
if equivalentMatch.ID != "" {
log.Debug(p.ctx, "Scanner: Found missing track with same base path", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(tx, equivalentMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name, err)
return err
}
p.totalMatched.Add(1)
} }
} }
return nil
}, "scanner: process missing tracks") // Use the exact match if found
if err != nil { if exactMatch.ID != "" {
return nil, err log.Debug(p.ctx, "Scanner: Found missing track in a new place", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(exactMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error moving matched track", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name, err)
return nil, err
}
p.totalMatched.Add(1)
continue
}
// If there is only one missing and one matched track, consider them equivalent (same PID)
if len(in.missing) == 1 && len(in.matched) == 1 {
singleMatch := in.matched[0]
log.Debug(p.ctx, "Scanner: Found track with same persistent ID in a new place", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(singleMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name, err)
return nil, err
}
p.totalMatched.Add(1)
continue
}
// Use the equivalent match if no other better match was found
if equivalentMatch.ID != "" {
log.Debug(p.ctx, "Scanner: Found missing track with same base path", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name)
err := p.moveMatched(equivalentMatch, ms)
if err != nil {
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name, err)
return nil, err
}
p.totalMatched.Add(1)
}
} }
return in, nil return in, nil
} }
func (p *phaseMissingTracks) moveMatched(tx model.DataStore, mt, ms model.MediaFile) error { func (p *phaseMissingTracks) moveMatched(mt, ms model.MediaFile) error {
discardedID := mt.ID return p.ds.WithTx(func(tx model.DataStore) error {
mt.ID = ms.ID discardedID := mt.ID
err := tx.MediaFile(p.ctx).Put(&mt) mt.ID = ms.ID
if err != nil { err := tx.MediaFile(p.ctx).Put(&mt)
return fmt.Errorf("update matched track: %w", err) if err != nil {
} return fmt.Errorf("update matched track: %w", err)
err = tx.MediaFile(p.ctx).Delete(discardedID) }
if err != nil { err = tx.MediaFile(p.ctx).Delete(discardedID)
return fmt.Errorf("delete discarded track: %w", err) if err != nil {
} return fmt.Errorf("delete discarded track: %w", err)
p.state.changesDetected.Store(true) }
return nil p.state.changesDetected.Store(true)
return nil
})
} }
func (p *phaseMissingTracks) finalize(err error) error { func (p *phaseMissingTracks) finalize(err error) error {

50
scanner/phase_3_refresh_albums.go
View file

@ -104,19 +104,13 @@ func (p *phaseRefreshAlbums) refreshAlbum(album *model.Album) (*model.Album, err
return nil, nil return nil, nil
} }
start := time.Now() start := time.Now()
err := p.ds.WithTx(func(tx model.DataStore) error { err := p.ds.Album(p.ctx).Put(album)
err := tx.Album(p.ctx).Put(album) log.Debug(p.ctx, "Scanner: refreshing album", "album_id", album.ID, "name", album.Name, "songCount", album.SongCount, "elapsed", time.Since(start), err)
log.Debug(p.ctx, "Scanner: refreshing album", "album_id", album.ID, "name", album.Name, "songCount", album.SongCount, "elapsed", time.Since(start))
if err != nil {
return fmt.Errorf("refreshing album %s: %w", album.ID, err)
}
p.refreshed.Add(1)
p.state.changesDetected.Store(true)
return nil
}, "scanner: refresh album")
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("refreshing album %s: %w", album.ID, err)
} }
p.refreshed.Add(1)
p.state.changesDetected.Store(true)
return album, nil return album, nil
} }
@ -135,23 +129,21 @@ func (p *phaseRefreshAlbums) finalize(err error) error {
log.Debug(p.ctx, "Scanner: No changes detected, skipping refreshing annotations") log.Debug(p.ctx, "Scanner: No changes detected, skipping refreshing annotations")
return nil return nil
} }
return p.ds.WithTx(func(tx model.DataStore) error { // Refresh album annotations
// Refresh album annotations start := time.Now()
start := time.Now() cnt, err := p.ds.Album(p.ctx).RefreshPlayCounts()
cnt, err := tx.Album(p.ctx).RefreshPlayCounts() if err != nil {
if err != nil { return fmt.Errorf("refreshing album annotations: %w", err)
return fmt.Errorf("refreshing album annotations: %w", err) }
} log.Debug(p.ctx, "Scanner: Refreshed album annotations", "albums", cnt, "elapsed", time.Since(start))
log.Debug(p.ctx, "Scanner: Refreshed album annotations", "albums", cnt, "elapsed", time.Since(start))
// Refresh artist annotations // Refresh artist annotations
start = time.Now() start = time.Now()
cnt, err = tx.Artist(p.ctx).RefreshPlayCounts() cnt, err = p.ds.Artist(p.ctx).RefreshPlayCounts()
if err != nil { if err != nil {
return fmt.Errorf("refreshing artist annotations: %w", err) return fmt.Errorf("refreshing artist annotations: %w", err)
} }
log.Debug(p.ctx, "Scanner: Refreshed artist annotations", "artists", cnt, "elapsed", time.Since(start)) log.Debug(p.ctx, "Scanner: Refreshed artist annotations", "artists", cnt, "elapsed", time.Since(start))
p.state.changesDetected.Store(true) p.state.changesDetected.Store(true)
return nil return nil
}, "scanner: finalize phaseRefreshAlbums")
} }

32
scanner/scanner.go
View file

@ -138,24 +138,22 @@ func (s *scannerImpl) runRefreshStats(ctx context.Context, state *scanState) fun
log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats") log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats")
return nil return nil
} }
return s.ds.WithTx(func(tx model.DataStore) error { start := time.Now()
start := time.Now() stats, err := s.ds.Artist(ctx).RefreshStats()
stats, err := tx.Artist(ctx).RefreshStats() if err != nil {
if err != nil { log.Error(ctx, "Scanner: Error refreshing artists stats", err)
log.Error(ctx, "Scanner: Error refreshing artists stats", err) return fmt.Errorf("refreshing artists stats: %w", err)
return fmt.Errorf("refreshing artists stats: %w", err) }
} log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
start = time.Now() start = time.Now()
err = tx.Tag(ctx).UpdateCounts() err = s.ds.Tag(ctx).UpdateCounts()
if err != nil { if err != nil {
log.Error(ctx, "Scanner: Error updating tag counts", err) log.Error(ctx, "Scanner: Error updating tag counts", err)
return fmt.Errorf("updating tag counts: %w", err) return fmt.Errorf("updating tag counts: %w", err)
} }
log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start)) log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start))
return nil return nil
}, "scanner: refresh stats")
} }
} }

2
server/nativeapi/playlists.go
View file

@ -100,7 +100,7 @@ func deleteFromPlaylist(ds model.DataStore) http.HandlerFunc {
p := req.Params(r) p := req.Params(r)
playlistId, _ := p.String(":playlistId") playlistId, _ := p.String(":playlistId")
ids, _ := p.Strings("id") ids, _ := p.Strings("id")
err := ds.WithTx(func(tx model.DataStore) error { err := ds.WithTxImmediate(func(tx model.DataStore) error {
tracksRepo := tx.Playlist(r.Context()).Tracks(playlistId, true) tracksRepo := tx.Playlist(r.Context()).Tracks(playlistId, true)
return tracksRepo.Delete(ids...) return tracksRepo.Delete(ids...)
}) })

2
server/subsonic/media_annotation.go
View file

@ -112,7 +112,7 @@ func (api *Router) setStar(ctx context.Context, star bool, ids ...string) error
return nil return nil
} }
event := &events.RefreshResource{} event := &events.RefreshResource{}
err := api.ds.WithTx(func(tx model.DataStore) error { err := api.ds.WithTxImmediate(func(tx model.DataStore) error {
for _, id := range ids { for _, id := range ids {
exist, err := tx.Album(ctx).Exists(id) exist, err := tx.Album(ctx).Exists(id)
if err != nil { if err != nil {

2
server/subsonic/playlists.go
View file

@ -58,7 +58,7 @@ func (api *Router) getPlaylist(ctx context.Context, id string) (*responses.Subso
} }
func (api *Router) create(ctx context.Context, playlistId, name string, ids []string) (string, error) { func (api *Router) create(ctx context.Context, playlistId, name string, ids []string) (string, error) {
err := api.ds.WithTx(func(tx model.DataStore) error { err := api.ds.WithTxImmediate(func(tx model.DataStore) error {
owner := getUser(ctx) owner := getUser(ctx)
var pls *model.Playlist var pls *model.Playlist
var err error var err error

4
tests/mock_data_store.go
View file

@ -213,6 +213,10 @@ func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...s
return block(db) return block(db)
} }
func (db *MockDataStore) WithTxImmediate(block func(tx model.DataStore) error, label ...string) error {
return block(db)
}
func (db *MockDataStore) Resource(context.Context, any) model.ResourceRepository { func (db *MockDataStore) Resource(context.Context, any) model.ResourceRepository {
return struct{ model.ResourceRepository }{} return struct{ model.ResourceRepository }{}
} }