refactor(server): replace RangeByChunks with Go 1.23 iterators (#3292)

* refactor(server): replace RangeByChunks with Go 1.23 iterators

* chore: fix comments re: SQLITE_MAX_VARIABLE_NUMBER

* test: improve playqueue test

* refactor(server): don't create a new iterator when it is not required
This commit is contained in:
Deluan Quintão 2024-09-22 11:47:10 -04:00 committed by GitHub
parent 3910e77a7a
commit 669c8f4c49
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 79 additions and 99 deletions

View file

@ -5,7 +5,6 @@ import (
"fmt"
"maps"
"path/filepath"
"slices"
"strings"
"time"
@ -72,9 +71,7 @@ func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity
return nil
}
ids := slices.Collect(maps.Keys(m))
chunks := slice.BreakUp(ids, 100)
for _, chunk := range chunks {
for chunk := range slice.CollectChunks(maps.Keys(m), 200) {
err := refresh(ctx, chunk...)
if err != nil {
log.Error(ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)

View file

@ -5,6 +5,7 @@ import (
"io/fs"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"time"
@ -20,7 +21,6 @@ import (
_ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg"
_ "github.com/navidrome/navidrome/scanner/metadata/taglib"
"github.com/navidrome/navidrome/utils/pl"
"github.com/navidrome/navidrome/utils/slice"
"golang.org/x/sync/errgroup"
)
@ -358,12 +358,11 @@ func (s *TagScanner) addOrUpdateTracksInDB(
currentTracks map[string]model.MediaFile,
filesToUpdate []string,
) (int, error) {
numUpdatedTracks := 0
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
numUpdatedTracks := 0
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
chunks := slice.BreakUp(filesToUpdate, filesBatchSize)
for _, chunk := range chunks {
for chunk := range slices.Chunk(filesToUpdate, filesBatchSize) {
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(chunk)
if err != nil {