diff --git a/.golangci.yml b/.golangci.yml index c8ffa330b..dcaeaaf65 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,4 +26,4 @@ issues: exclude-rules: - linters: - gosec - text: "(G501|G401):" + text: "(G501|G401|G505):" diff --git a/core/cache/file_caches.go b/core/cache/file_caches.go index 2264e933e..127ee80b0 100644 --- a/core/cache/file_caches.go +++ b/core/cache/file_caches.go @@ -184,7 +184,7 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach cacheFolder = filepath.Join(conf.Server.DataFolder, cacheFolder) log.Info(fmt.Sprintf("Creating %s cache", name), "path", cacheFolder, "maxSize", humanize.Bytes(size)) - fs, err := NewSpreadFs(cacheFolder, 0755) + fs, err := NewSpreadFS(cacheFolder, 0755) if err != nil { log.Error(fmt.Sprintf("Error initializing %s cache", name), err, "elapsedTime", time.Since(start)) return nil, err diff --git a/core/cache/spread_fs.go b/core/cache/spread_fs.go index 755b5a599..813ca79f7 100644 --- a/core/cache/spread_fs.go +++ b/core/cache/spread_fs.go @@ -1,8 +1,9 @@ package cache import ( - "crypto/md5" + "crypto/sha1" "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -13,22 +14,24 @@ import ( "gopkg.in/djherbis/stream.v1" ) -type spreadFs struct { +type spreadFS struct { root string mode os.FileMode init func() error } -// NewSpreadFs returns a FileSystem rooted at directory dir. It +const keyFileExtension = ".key" + +// NewSpreadFS returns a FileSystem rooted at directory dir. It // Dir is created with perms if it doesn't exist. -func NewSpreadFs(dir string, mode os.FileMode) (fscache.FileSystem, error) { - fs := &spreadFs{root: dir, mode: mode, init: func() error { +func NewSpreadFS(dir string, mode os.FileMode) (fscache.FileSystem, error) { + fs := &spreadFS{root: dir, mode: mode, init: func() error { return os.MkdirAll(dir, mode) }} return fs, fs.init() } -func (fs *spreadFs) Reload(f func(key string, name string)) error { +func (fs *spreadFS) Reload(f func(key string, name string)) error { return godirwalk.Walk(fs.root, &godirwalk.Options{ Callback: func(absoluteFilePath string, de *godirwalk.Dirent) error { path, err := filepath.Rel(fs.root, absoluteFilePath) @@ -36,38 +39,63 @@ func (fs *spreadFs) Reload(f func(key string, name string)) error { return nil } + // Skip if name is not in the format XX/XX/XXXXXXXXXXXX.key parts := strings.Split(path, string(os.PathSeparator)) - if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 { + if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || + filepath.Ext(path) != keyFileExtension { return nil } - key := filepath.Base(path) - f(key, absoluteFilePath) + keyFileName := absoluteFilePath + dataFileName := absoluteFilePath[0 : len(absoluteFilePath)-len(keyFileExtension)] + + // Load the key from the key file. Remove and skip on error + key, err := ioutil.ReadFile(keyFileName) + if err != nil { + _ = fs.Remove(dataFileName) + return nil + } + + // If the data file is not readable, remove and skip + file, err := os.Open(dataFileName) + defer func() { _ = file.Close() }() + if err != nil { + _ = fs.Remove(dataFileName) + return nil + } + + f(string(key), dataFileName) return nil }, Unsorted: true, }) } -func (fs *spreadFs) Create(name string) (stream.File, error) { - key := fmt.Sprintf("%x", md5.Sum([]byte(name))) +func (fs *spreadFS) Create(name string) (stream.File, error) { + key := fmt.Sprintf("%x", sha1.Sum([]byte(name))) path := fmt.Sprintf("%s%c%s", key[0:2], os.PathSeparator, key[2:4]) err := os.MkdirAll(filepath.Join(fs.root, path), fs.mode) if err != nil { return nil, err } - return os.OpenFile(filepath.Join(fs.root, path, key), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + absolutePath := filepath.Join(fs.root, path, key) + err = ioutil.WriteFile(absolutePath+keyFileExtension, []byte(name), 0600) + if err != nil { + return nil, err + } + return os.OpenFile(absolutePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) } -func (fs *spreadFs) Open(name string) (stream.File, error) { +func (fs *spreadFS) Open(name string) (stream.File, error) { return os.Open(name) } -func (fs *spreadFs) Remove(name string) error { +func (fs *spreadFS) Remove(name string) error { + _ = os.Remove(name + keyFileExtension) return os.Remove(name) } -func (fs *spreadFs) Stat(name string) (fscache.FileInfo, error) { +func (fs *spreadFS) Stat(name string) (fscache.FileInfo, error) { stat, err := os.Stat(name) if err != nil { return fscache.FileInfo{}, err @@ -75,7 +103,7 @@ func (fs *spreadFs) Stat(name string) (fscache.FileInfo, error) { return fscache.FileInfo{FileInfo: stat, Atime: atime.Get(stat)}, nil } -func (fs *spreadFs) RemoveAll() error { +func (fs *spreadFS) RemoveAll() error { if err := os.RemoveAll(fs.root); err != nil { return err }