New Cache FileSystem implementation

core/artwork.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/deluan/navidrome/core/cache"
 	_ "golang.org/x/image/webp"
 
 	"github.com/deluan/navidrome/conf"
@@ -30,7 +31,7 @@ type Artwork interface {
 	Get(ctx context.Context, id string, size int, out io.Writer) error
 }
 
-type ArtworkCache FileCache
+type ArtworkCache cache.FileCache
 
 func NewArtwork(ds model.DataStore, cache ArtworkCache) Artwork {
 	return &artwork{ds: ds, cache: cache}
@@ -38,7 +39,7 @@ func NewArtwork(ds model.DataStore, cache ArtworkCache) Artwork {
 
 type artwork struct {
 	ds    model.DataStore
-	cache FileCache
+	cache cache.FileCache
 }
 
 type imageInfo struct {
@@ -196,7 +197,7 @@ func readFromFile(path string) ([]byte, error) {
 }
 
 func NewImageCache() ArtworkCache {
-	return NewFileCache("Image", conf.Server.ImageCacheSize, consts.ImageCacheDir, consts.DefaultImageCacheMaxItems,
+	return cache.NewFileCache("Image", conf.Server.ImageCacheSize, consts.ImageCacheDir, consts.DefaultImageCacheMaxItems,
 		func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
 			info := arg.(*imageInfo)
 			reader, err := info.c.getArtwork(ctx, info.path, info.size)

core/cache/cache_suite_test.go

@@ -0,0 +1,17 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/tests"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCache(t *testing.T) {
+	tests.Init(t, false)
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Cache Suite")
+}

core/cache/file_caches.go

@@ -1,4 +1,4 @@
-package core
+package cache
 
 import (
 	"context"
@@ -184,7 +184,7 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach
 	cacheFolder = filepath.Join(conf.Server.DataFolder, cacheFolder)
 
 	log.Info(fmt.Sprintf("Creating %s cache", name), "path", cacheFolder, "maxSize", humanize.Bytes(size))
-	fs, err := fscache.NewFs(cacheFolder, 0755)
+	fs, err := NewSpreadFs(cacheFolder, 0755)
 	if err != nil {
 		log.Error(fmt.Sprintf("Error initializing %s cache", name), err, "elapsedTime", time.Since(start))
 		return nil, err

core/cache/file_caches_test.go

@@ -1,4 +1,4 @@
-package core
+package cache
 
 import (
 	"context"

core/cache/spread_fs.go

@@ -0,0 +1,83 @@
+package cache
+
+import (
+	"crypto/md5"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/djherbis/fscache"
+	"github.com/karrick/godirwalk"
+	"gopkg.in/djherbis/atime.v1"
+	"gopkg.in/djherbis/stream.v1"
+)
+
+type spreadFs struct {
+	root string
+	mode os.FileMode
+	init func() error
+}
+
+// NewSpreadFs returns a FileSystem rooted at directory dir. It
+// Dir is created with perms if it doesn't exist.
+func NewSpreadFs(dir string, mode os.FileMode) (fscache.FileSystem, error) {
+	fs := &spreadFs{root: dir, mode: mode, init: func() error {
+		return os.MkdirAll(dir, mode)
+	}}
+	return fs, fs.init()
+}
+
+func (fs *spreadFs) Reload(f func(key string, name string)) error {
+	return godirwalk.Walk(fs.root, &godirwalk.Options{
+		Callback: func(absoluteFilePath string, de *godirwalk.Dirent) error {
+			path, err := filepath.Rel(fs.root, absoluteFilePath)
+			if err != nil {
+				return nil
+			}
+
+			parts := strings.Split(path, string(os.PathSeparator))
+			if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 {
+				return nil
+			}
+
+			key := filepath.Base(path)
+			f(key, absoluteFilePath)
+			return nil
+		},
+		Unsorted: true,
+	})
+}
+
+func (fs *spreadFs) Create(name string) (stream.File, error) {
+	key := fmt.Sprintf("%x", md5.Sum([]byte(name)))
+	path := fmt.Sprintf("%s%c%s", key[0:2], os.PathSeparator, key[2:4])
+	err := os.MkdirAll(filepath.Join(fs.root, path), fs.mode)
+	if err != nil {
+		return nil, err
+	}
+	return os.OpenFile(filepath.Join(fs.root, path, key), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+}
+
+func (fs *spreadFs) Open(name string) (stream.File, error) {
+	return os.Open(name)
+}
+
+func (fs *spreadFs) Remove(name string) error {
+	return os.Remove(name)
+}
+
+func (fs *spreadFs) Stat(name string) (fscache.FileInfo, error) {
+	stat, err := os.Stat(name)
+	if err != nil {
+		return fscache.FileInfo{}, err
+	}
+	return fscache.FileInfo{FileInfo: stat, Atime: atime.Get(stat)}, nil
+}
+
+func (fs *spreadFs) RemoveAll() error {
+	if err := os.RemoveAll(fs.root); err != nil {
+		return err
+	}
+	return fs.init()
+}

core/media_streamer.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/core/cache"
 	"github.com/deluan/navidrome/core/transcoder"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
@@ -20,7 +21,7 @@ type MediaStreamer interface {
 	NewStream(ctx context.Context, id string, reqFormat string, reqBitRate int) (*Stream, error)
 }
 
-type TranscodingCache FileCache
+type TranscodingCache cache.FileCache
 
 func NewMediaStreamer(ds model.DataStore, ffm transcoder.Transcoder, cache TranscodingCache) MediaStreamer {
 	return &mediaStreamer{ds: ds, ffm: ffm, cache: cache}
@@ -29,7 +30,7 @@ func NewMediaStreamer(ds model.DataStore, ffm transcoder.Transcoder, cache Trans
 type mediaStreamer struct {
 	ds    model.DataStore
 	ffm   transcoder.Transcoder
-	cache FileCache
+	cache cache.FileCache
 }
 
 type streamJob struct {
@@ -167,7 +168,7 @@ func selectTranscodingOptions(ctx context.Context, ds model.DataStore, mf *model
 }
 
 func NewTranscodingCache() TranscodingCache {
-	return NewFileCache("Transcoding", conf.Server.TranscodingCacheSize,
+	return cache.NewFileCache("Transcoding", conf.Server.TranscodingCacheSize,
 		consts.TranscodingCacheDir, consts.DefaultTranscodingCacheMaxItems,
 		func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
 			job := arg.(*streamJob)

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/golangci/golangci-lint v1.31.0
 	github.com/google/uuid v1.1.2
 	github.com/google/wire v0.4.0
+	github.com/karrick/godirwalk v1.16.1
 	github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629
 	github.com/lib/pq v1.3.0 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
@@ -43,8 +44,8 @@ require (
 	golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
 	golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0
 	google.golang.org/protobuf v1.25.0 // indirect
-	gopkg.in/djherbis/atime.v1 v1.0.0 // indirect
-	gopkg.in/djherbis/stream.v1 v1.3.1 // indirect
+	gopkg.in/djherbis/atime.v1 v1.0.0
+	gopkg.in/djherbis/stream.v1 v1.3.1
 	gopkg.in/ini.v1 v1.57.0 // indirect
 )
 

go.sum

@@ -293,6 +293,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629 h1:m1E9veL+2sjZOMSM7y3a6jJ9fNVaGyIJCXYDPm9U+/0=