New Folder Scanner - WIP

.gitignore

@@ -10,4 +10,4 @@ Artwork
 sonic.toml
 master.zip
 Jamstash-master
-storm.db
+testDB

Makefile

@@ -21,7 +21,7 @@ test: check_go_env
 .PHONY: build
 build: check_go_env
 	go build
-	@(cd ./ui && npm run build)
+#	@(cd ./ui && npm run build)
 
 .PHONY: setup
 setup: Jamstash-master

conf/configuration.go

@@ -20,6 +20,7 @@ type sonic struct {
 
 	DisableDownsampling bool   `default:"false"`
 	DownsampleCommand   string `default:"ffmpeg -i %s -map 0:0 -b:a %bk -v 0 -f mp3 -"`
+	ProbeCommand        string `default:"ffprobe -v quiet -print_format json -show_format %s"`
 	PlsIgnoreFolders    bool   `default:"true"`
 	PlsIgnoredPatterns  string `default:"^iCloud;\\~"`
 
@@ -28,6 +29,7 @@ type sonic struct {
 	DevDisableAuthentication bool   `default:"false"`
 	DevDisableFileCheck      bool   `default:"false"`
 	DevDisableBanner         bool   `default:"false"`
+	DevUseFileScanner        bool   `default:"false"`
 }
 
 var Sonic *sonic

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/Masterminds/squirrel v1.1.0
 	github.com/astaxie/beego v1.12.0
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
-	github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4
 	github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131
 	github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a // indirect
 	github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8
@@ -15,7 +14,6 @@ require (
 	github.com/fatih/structs v1.0.0 // indirect
 	github.com/go-chi/chi v4.0.3+incompatible
 	github.com/go-chi/cors v1.0.0
-	github.com/google/uuid v1.1.1
 	github.com/google/wire v0.4.0
 	github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629
 	github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a

go.sum

@@ -22,14 +22,10 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4 h1:VSoAcWJvj656TSyWbJ5KuGsi/J8dO5+iO9+5/7I8wao=
-github.com/deluan/rest v0.0.0-20200114062534-0653ffe9eab4/go.mod h1:tSgDythFsl0QgS/PFWfIZqcJKnkADWneY80jaVRlqK8=
 github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131 h1:siEGb+iB1Ea75U7BnkYVSqSRzE6QHlXCbqEXenxRmhQ=
 github.com/dhowden/itl v0.0.0-20170329215456-9fbe21093131/go.mod h1:eVWQJVQ67aMvYhpkDwaH2Goy2vo6v8JCMfGXfQ9sPtw=
 github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a h1:7MucP9rMAsQRcRE1sGpvMZoTxFYZlDmfDvCH+z7H+90=
 github.com/dhowden/plist v0.0.0-20141002110153-5db6e0d9931a/go.mod h1:sLjdR6uwx3L6/Py8F+QgAfeiuY87xuYGwCDqRFrvCzw=
-github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10 h1:H4waV2XHwsud7v90IfEbwwynwVlC7YU3RicLlfkfgn4=
-github.com/dhowden/tag v0.0.0-20170128231422-9edd38ca5d10/go.mod h1:SniNVYuaD1jmdEEvi+7ywb1QFR7agjeTdGKyFb0p7Rw=
 github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8 h1:nmFnmD8VZXkjDHIb1Gnfz50cgzUvGN72zLjPRXBW/hU=
 github.com/dhowden/tag v0.0.0-20191122115059-7e5c04feccd8/go.mod h1:SniNVYuaD1jmdEEvi+7ywb1QFR7agjeTdGKyFb0p7Rw=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
@@ -58,8 +54,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
 github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -84,8 +78,6 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtB
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=

log/log.go

@@ -132,7 +132,7 @@ func parseArgs(args []interface{}) (*logrus.Entry, string) {
 		kvPairs := args[1:]
 		l = addFields(l, kvPairs)
 	}
-	if currentLevel >= LevelDebug {
+	if currentLevel >= LevelTrace {
 		_, file, line, ok := runtime.Caller(2)
 		if !ok {
 			file = "???"

model/album.go

@@ -37,4 +37,5 @@ type AlbumRepository interface {
 	GetAllIds() ([]string, error)
 	GetStarred(...QueryOptions) (Albums, error)
 	Search(q string, offset int, size int) (Albums, error)
+	Refresh(ids ...string) error
 }

model/artist.go

@@ -13,6 +13,7 @@ type ArtistRepository interface {
 	Get(id string) (*Artist, error)
 	PurgeInactive(active Artists) error
 	Search(q string, offset int, size int) (Artists, error)
+	Refresh(ids ...string) error
 }
 
 type Artists []Artist

model/index.go

@@ -14,11 +14,10 @@ type ArtistIndex struct {
 type ArtistInfos []ArtistInfo
 type ArtistIndexes []ArtistIndex
 
+// TODO Combine ArtistIndex with Artist
 type ArtistIndexRepository interface {
-	CountAll() (int64, error)
-	Exists(id string) (bool, error)
 	Put(m *ArtistIndex) error
-	Get(id string) (*ArtistIndex, error)
+	Refresh() error
 	GetAll() (ArtistIndexes, error)
 	DeleteAll() error
 }

model/mediafile.go

@@ -51,4 +51,5 @@ type MediaFileRepository interface {
 	GetAllIds() ([]string, error)
 	Search(q string, offset int, size int) (MediaFiles, error)
 	Delete(id string) error
+	DeleteByPath(path string) error
 }

persistence/album_repository.go

@@ -1,9 +1,12 @@
 package persistence
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/astaxie/beego/orm"
+	"github.com/cloudsonic/sonic-server/log"
 	"github.com/cloudsonic/sonic-server/model"
 )
 
@@ -85,7 +88,62 @@ func (r *albumRepository) toAlbums(all []Album) model.Albums {
 	return result
 }
 
-// TODO Remove []string from return
+func (r *albumRepository) Refresh(ids ...string) error {
+	type refreshAlbum struct {
+		Album
+		CurrentId   string
+		HasCoverArt bool
+	}
+	var albums []refreshAlbum
+	o := Db()
+	sql := fmt.Sprintf(`
+select album_id as id, album as name, f.artist, f.album_artist, f.artist_id, f.compilation, f.genre,  
+	max(f.year) as year, sum(f.play_count) as play_count, max(f.play_date) as play_date,  sum(f.duration) as duration,
+	max(f.updated_at) as updated_at, min(f.created_at) as created_at, count(*) as song_count, 
+	a.id as current_id, f.id as cover_art_id, f.path as cover_art_path, f.has_cover_art
+from media_file f left outer join album a on f.album_id = a.id
+where f.album_id in ('%s')
+group by album_id order by f.id`, strings.Join(ids, "','"))
+	_, err := o.Raw(sql).QueryRows(&albums)
+	if err != nil {
+		return err
+	}
+
+	var toInsert []Album
+	var toUpdate []Album
+	for _, al := range albums {
+		if !al.HasCoverArt {
+			al.CoverArtId = ""
+		}
+		if al.Compilation {
+			al.AlbumArtist = "Various Artists"
+		}
+		if al.CurrentId != "" {
+			toUpdate = append(toUpdate, al.Album)
+		} else {
+			toInsert = append(toInsert, al.Album)
+		}
+	}
+	if len(toInsert) > 0 {
+		n, err := o.InsertMulti(100, toInsert)
+		if err != nil {
+			return err
+		}
+		log.Debug("Inserted new albums", "num", n)
+	}
+	if len(toUpdate) > 0 {
+		for _, al := range toUpdate {
+			_, err := o.Update(&al, "name", "artist_id", "cover_art_path", "cover_art_id", "artist", "album_artist", "year",
+				"compilation", "play_count", "song_count", "duration", "updated_at")
+			if err != nil {
+				return err
+			}
+		}
+		log.Debug("Updated albums", "num", len(toUpdate))
+	}
+	return err
+}
+
 func (r *albumRepository) PurgeInactive(activeList model.Albums) error {
 	return withTx(func(o orm.Ormer) error {
 		_, err := r.purgeInactive(o, activeList, func(item interface{}) string {

persistence/artist_repository.go

@@ -1,7 +1,11 @@
 package persistence
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/astaxie/beego/orm"
+	"github.com/cloudsonic/sonic-server/log"
 	"github.com/cloudsonic/sonic-server/model"
 )
 
@@ -42,6 +46,64 @@ func (r *artistRepository) Get(id string) (*model.Artist, error) {
 	return &a, nil
 }
 
+func (r *artistRepository) Refresh(ids ...string) error {
+	type refreshArtist struct {
+		Artist
+		CurrentId   string
+		AlbumArtist string
+		Compilation bool
+	}
+	var artists []refreshArtist
+	o := Db()
+	sql := fmt.Sprintf(`
+select f.artist_id as id,
+       f.artist as name,
+       f.album_artist,
+       f.compilation,
+       count(*) as album_count,
+       a.id as current_id
+from album f
+         left outer join artist a on f.artist_id = a.id
+where f.artist_id in ('%s') group by f.artist_id order by f.id`, strings.Join(ids, "','"))
+	_, err := o.Raw(sql).QueryRows(&artists)
+	if err != nil {
+		return err
+	}
+
+	var toInsert []Artist
+	var toUpdate []Artist
+	for _, al := range artists {
+		if al.Compilation {
+			al.AlbumArtist = "Various Artists"
+		}
+		if al.AlbumArtist != "" {
+			al.Name = al.AlbumArtist
+		}
+		if al.CurrentId != "" {
+			toUpdate = append(toUpdate, al.Artist)
+		} else {
+			toInsert = append(toInsert, al.Artist)
+		}
+	}
+	if len(toInsert) > 0 {
+		n, err := o.InsertMulti(100, toInsert)
+		if err != nil {
+			return err
+		}
+		log.Debug("Inserted new artists", "num", n)
+	}
+	if len(toUpdate) > 0 {
+		for _, al := range toUpdate {
+			_, err := o.Update(&al, "name", "album_count")
+			if err != nil {
+				return err
+			}
+		}
+		log.Debug("Updated artists", "num", len(toUpdate))
+	}
+	return err
+}
+
 func (r *artistRepository) PurgeInactive(activeList model.Artists) error {
 	return withTx(func(o orm.Ormer) error {
 		_, err := r.purgeInactive(o, activeList, func(item interface{}) string {

persistence/index_repository.go

@@ -2,9 +2,12 @@ package persistence
 
 import (
 	"sort"
+	"strings"
 
 	"github.com/astaxie/beego/orm"
+	"github.com/cloudsonic/sonic-server/conf"
 	"github.com/cloudsonic/sonic-server/model"
+	"github.com/cloudsonic/sonic-server/utils"
 )
 
 type ArtistInfo struct {
@@ -15,6 +18,8 @@ type ArtistInfo struct {
 	AlbumCount int
 }
 
+type tempIndex map[string]model.ArtistInfo
+
 type artistIndexRepository struct {
 	sqlRepository
 }
@@ -25,15 +30,6 @@ func NewArtistIndexRepository() model.ArtistIndexRepository {
 	return r
 }
 
-func (r *artistIndexRepository) CountAll() (int64, error) {
-	count := struct{ Count int64 }{}
-	err := Db().Raw("select count(distinct(idx)) as count from artist_info").QueryRow(&count)
-	if err != nil {
-		return 0, err
-	}
-	return count.Count, nil
-}
-
 func (r *artistIndexRepository) Put(idx *model.ArtistIndex) error {
 	return withTx(func(o orm.Ormer) error {
 		_, err := r.newQuery(o).Filter("idx", idx.ID).Delete()
@@ -56,23 +52,64 @@ func (r *artistIndexRepository) Put(idx *model.ArtistIndex) error {
 	})
 }
 
-func (r *artistIndexRepository) Get(id string) (*model.ArtistIndex, error) {
+func (r *artistIndexRepository) Refresh() error {
-	var ais []ArtistInfo
+	o := Db()
-	_, err := r.newQuery(Db()).Filter("idx", id).All(&ais)
+
+	indexGroups := utils.ParseIndexGroups(conf.Sonic.IndexGroups)
+	artistIndex := make(map[string]tempIndex)
+
+	var artists []Artist
+	_, err := o.QueryTable(&Artist{}).All(&artists)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	idx := &model.ArtistIndex{ID: id}
+	for _, ar := range artists {
-	idx.Artists = make([]model.ArtistInfo, len(ais))
+		r.collectIndex(indexGroups, &ar, artistIndex)
-	for i, a := range ais {
+	}
-		idx.Artists[i] = model.ArtistInfo{
+
-			ArtistID:   a.ArtistID,
+	return r.saveIndex(artistIndex)
-			Artist:     a.Artist,
+}
-			AlbumCount: a.AlbumCount,
+
+func (r *artistIndexRepository) collectIndex(ig utils.IndexGroups, a *Artist, artistIndex map[string]tempIndex) {
+	name := a.Name
+	indexName := strings.ToLower(utils.NoArticle(name))
+	if indexName == "" {
+		return
+	}
+	group := r.findGroup(ig, indexName)
+	artists := artistIndex[group]
+	if artists == nil {
+		artists = make(tempIndex)
+		artistIndex[group] = artists
+	}
+	artists[indexName] = model.ArtistInfo{ArtistID: a.ID, Artist: a.Name, AlbumCount: a.AlbumCount}
+}
+
+func (r *artistIndexRepository) findGroup(ig utils.IndexGroups, name string) string {
+	for k, v := range ig {
+		key := strings.ToLower(k)
+		if strings.HasPrefix(name, key) {
+			return v
 		}
 	}
-	return idx, err
+	return "#"
+}
+
+func (r *artistIndexRepository) saveIndex(artistIndex map[string]tempIndex) error {
+	r.DeleteAll()
+	for k, temp := range artistIndex {
+		idx := &model.ArtistIndex{ID: k}
+		for _, v := range temp {
+			idx.Artists = append(idx.Artists, v)
+		}
+		err := r.Put(idx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (r *artistIndexRepository) GetAll() (model.ArtistIndexes, error) {

persistence/index_repository_test.go

@@ -35,11 +35,6 @@ var _ = Describe("Artist Index", func() {
 
 		Expect(repo.Put(&idx1)).To(BeNil())
 		Expect(repo.Put(&idx2)).To(BeNil())
-		Expect(repo.Get("D")).To(Equal(&idx1))
-		Expect(repo.Get("S")).To(Equal(&idx2))
 		Expect(repo.GetAll()).To(Equal(model.ArtistIndexes{idx1, idx2}))
-		Expect(repo.CountAll()).To(Equal(int64(2)))
-		Expect(repo.DeleteAll()).To(BeNil())
-		Expect(repo.CountAll()).To(Equal(int64(0)))
 	})
 })

persistence/mediafile_repository.go

@@ -102,6 +102,26 @@ func (r *mediaFileRepository) FindByPath(path string) (model.MediaFiles, error)
 	return r.toMediaFiles(filtered), nil
 }
 
+func (r *mediaFileRepository) DeleteByPath(path string) error {
+	o := Db()
+	var mfs []MediaFile
+	_, err := r.newQuery(o).Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs)
+	if err != nil {
+		return err
+	}
+	var filtered []string
+	path = strings.ToLower(path) + string(os.PathSeparator)
+	for _, mf := range mfs {
+		filename := strings.TrimPrefix(strings.ToLower(mf.Path), path)
+		if len(strings.Split(filename, string(os.PathSeparator))) > 1 {
+			continue
+		}
+		filtered = append(filtered, mf.ID)
+	}
+	_, err = r.newQuery(o).Filter("id__in", filtered).Delete()
+	return err
+}
+
 func (r *mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) {
 	var starred []MediaFile
 	_, err := r.newQuery(Db(), options...).Filter("starred", true).All(&starred)

scanner/change_detector.go

@@ -0,0 +1,113 @@
+package scanner
+
+import (
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/cloudsonic/sonic-server/log"
+)
+
+type ChangeDetector struct {
+	rootFolder string
+	lastUpdate time.Time
+	dirMap     map[string]time.Time
+}
+
+func NewChangeDetector(rootFolder string, lastUpdate time.Time) *ChangeDetector {
+	return &ChangeDetector{
+		rootFolder: rootFolder,
+		lastUpdate: lastUpdate,
+		dirMap:     map[string]time.Time{},
+	}
+}
+
+func (s *ChangeDetector) Scan() (changed []string, deleted []string, err error) {
+	start := time.Now()
+	newMap := make(map[string]time.Time)
+	err = s.loadMap(s.rootFolder, newMap)
+	if err != nil {
+		return
+	}
+	changed, deleted, err = s.checkForUpdates(s.dirMap, newMap)
+	if err != nil {
+		return
+	}
+	elapsed := time.Since(start)
+
+	log.Trace("Folder analysis complete\n", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed)
+	s.dirMap = newMap
+	return
+}
+
+func (s *ChangeDetector) loadDir(dirPath string) (children []string, lastUpdated time.Time, err error) {
+	dir, err := os.Open(dirPath)
+	if err != nil {
+		return
+	}
+	dirInfo, err := os.Stat(dirPath)
+	if err != nil {
+		return
+	}
+	lastUpdated = dirInfo.ModTime()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return
+	}
+	for _, f := range files {
+		if f.IsDir() {
+			children = append(children, path.Join(dirPath, f.Name()))
+		} else {
+			if f.ModTime().After(lastUpdated) {
+				lastUpdated = f.ModTime()
+			}
+		}
+	}
+	return
+}
+
+func (s *ChangeDetector) loadMap(rootPath string, dirMap map[string]time.Time) error {
+	children, lastUpdated, err := s.loadDir(rootPath)
+	if err != nil {
+		return err
+	}
+	for _, c := range children {
+		err := s.loadMap(c, dirMap)
+		if err != nil {
+			return err
+		}
+	}
+
+	dir := s.getRelativePath(rootPath)
+	dirMap[dir] = lastUpdated
+
+	return nil
+}
+
+func (s *ChangeDetector) getRelativePath(subfolder string) string {
+	dir := strings.TrimPrefix(subfolder, s.rootFolder)
+	if dir == "" {
+		dir = "."
+	}
+	return dir
+}
+
+func (s *ChangeDetector) checkForUpdates(oldMap map[string]time.Time, newMap map[string]time.Time) (changed []string, deleted []string, err error) {
+	for dir, lastUpdated := range newMap {
+		oldLastUpdated, ok := oldMap[dir]
+		if !ok {
+			oldLastUpdated = s.lastUpdate
+		}
+		if lastUpdated.After(oldLastUpdated) {
+			changed = append(changed, dir)
+		}
+	}
+	for dir := range oldMap {
+		if _, ok := newMap[dir]; !ok {
+			deleted = append(deleted, dir)
+		}
+	}
+	return
+}

scanner/change_detector_test.go

@@ -0,0 +1,103 @@
+package scanner
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ChangeDetector", func() {
+	var testFolder string
+	var scanner *ChangeDetector
+
+	BeforeEach(func() {
+		testFolder, _ = ioutil.TempDir("", "cloudsonic_tests")
+		err := os.MkdirAll(testFolder, 0700)
+		if err != nil {
+			panic(err)
+		}
+		scanner = NewChangeDetector(testFolder, time.Time{})
+	})
+
+	It("detects changes recursively", func() {
+		// Scan empty folder
+		changed, deleted, err := scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf("."))
+
+		// Add one subfolder
+		err = os.MkdirAll(path.Join(testFolder, "a"), 0700)
+		if err != nil {
+			panic(err)
+		}
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf(".", "/a"))
+
+		// Add more subfolders
+		err = os.MkdirAll(path.Join(testFolder, "a", "b", "c"), 0700)
+		if err != nil {
+			panic(err)
+		}
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf("/a", "/a/b", "/a/b/c"))
+
+		// Scan with no changes
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(BeEmpty())
+
+		// New file in subfolder
+		_, err = os.Create(path.Join(testFolder, "a", "b", "empty.txt"))
+		if err != nil {
+			panic(err)
+		}
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf("/a/b"))
+
+		// Delete file in subfolder
+		err = os.Remove(path.Join(testFolder, "a", "b", "empty.txt"))
+		if err != nil {
+			panic(err)
+		}
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf("/a/b"))
+
+		// Delete subfolder
+		err = os.Remove(path.Join(testFolder, "a", "b", "c"))
+		if err != nil {
+			panic(err)
+		}
+		changed, deleted, err = scanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(ConsistOf("/a/b/c"))
+		Expect(changed).To(ConsistOf("/a/b"))
+
+		// Only returns changes after lastUpdate
+		newScanner := NewChangeDetector(testFolder, time.Now())
+		changed, deleted, err = newScanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(BeEmpty())
+		Expect(changed).To(BeEmpty())
+
+		_, err = os.Create(path.Join(testFolder, "a", "b", "new.txt"))
+		changed, deleted, err = newScanner.Scan()
+		Expect(err).To(BeNil())
+		Expect(deleted).To(BeEmpty())
+		Expect(changed).To(ConsistOf("/a/b"))
+	})
+})

scanner/metadata.go

@@ -0,0 +1,140 @@
+package scanner
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudsonic/sonic-server/conf"
+	"github.com/cloudsonic/sonic-server/log"
+	"github.com/dhowden/tag"
+)
+
+type Metadata struct {
+	filePath    string
+	suffix      string
+	fileInfo    os.FileInfo
+	t           tag.Metadata
+	duration    int
+	bitRate     int
+	compilation bool
+}
+
+func ExtractMetadata(filePath string) (*Metadata, error) {
+	m := &Metadata{filePath: filePath}
+	m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
+	fi, err := os.Stat(filePath)
+	if err != nil {
+		return nil, err
+	}
+	m.fileInfo = fi
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	t, err := tag.ReadFrom(f)
+	if err != nil {
+		return nil, err
+	}
+	m.t = t
+
+	err = m.probe(filePath)
+	return m, err
+}
+
+func (m *Metadata) Title() string               { return m.t.Title() }
+func (m *Metadata) Album() string               { return m.t.Album() }
+func (m *Metadata) Artist() string              { return m.t.Artist() }
+func (m *Metadata) AlbumArtist() string         { return m.t.AlbumArtist() }
+func (m *Metadata) Composer() string            { return m.t.Composer() }
+func (m *Metadata) Genre() string               { return m.t.Genre() }
+func (m *Metadata) Year() int                   { return m.t.Year() }
+func (m *Metadata) TrackNumber() (int, int)     { return m.t.Track() }
+func (m *Metadata) DiscNumber() (int, int)      { return m.t.Disc() }
+func (m *Metadata) HasPicture() bool            { return m.t.Picture() != nil }
+func (m *Metadata) Compilation() bool           { return m.compilation }
+func (m *Metadata) Duration() int               { return m.duration }
+func (m *Metadata) BitRate() int                { return m.bitRate }
+func (m *Metadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
+func (m *Metadata) FilePath() string            { return m.filePath }
+func (m *Metadata) Suffix() string              { return m.suffix }
+func (m *Metadata) Size() int                   { return int(m.fileInfo.Size()) }
+
+// probe analyzes the file and returns duration in seconds and bitRate in kb/s.
+// It uses the ffprobe external tool, configured in conf.Sonic.ProbeCommand
+func (m *Metadata) probe(filePath string) error {
+	cmdLine, args := createProbeCommand(filePath)
+
+	log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
+	cmd := exec.Command(cmdLine, args...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return err
+	}
+	return m.parseOutput(output)
+}
+
+func (m *Metadata) parseInt(objItf interface{}, field string) (int, error) {
+	obj := objItf.(map[string]interface{})
+	s, ok := obj[field].(string)
+	if !ok {
+		return -1, errors.New("invalid ffprobe output field obj." + field)
+	}
+	fDuration, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return -1, err
+	}
+	return int(fDuration), nil
+}
+
+func (m *Metadata) parseOutput(output []byte) error {
+	var data map[string]map[string]interface{}
+	err := json.Unmarshal(output, &data)
+	if err != nil {
+		return err
+	}
+
+	format, ok := data["format"]
+	if !ok {
+		err = errors.New("invalid ffprobe output. no format found")
+		return err
+	}
+
+	if tags, ok := format["tags"]; ok {
+		c, _ := m.parseInt(tags, "compilation")
+		m.compilation = c == 1
+	}
+
+	m.duration, err = m.parseInt(format, "duration")
+	if err != nil {
+		return err
+	}
+
+	m.bitRate, err = m.parseInt(format, "bit_rate")
+	m.bitRate = m.bitRate / 1000
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func createProbeCommand(filePath string) (string, []string) {
+	cmd := conf.Sonic.ProbeCommand
+
+	split := strings.Split(cmd, " ")
+	for i, s := range split {
+		s = strings.Replace(s, "%s", filePath, -1)
+		split[i] = s
+	}
+
+	return split[0], split[1:]
+}

scanner/metadata_test.go

@@ -0,0 +1,55 @@
+package scanner
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Metadata", func() {
+	It("correctly parses mp3 file", func() {
+		m, err := ExtractMetadata("../tests/fixtures/test.mp3")
+		Expect(err).To(BeNil())
+		Expect(m.Title()).To(Equal("Song"))
+		Expect(m.Album()).To(Equal("Album"))
+		Expect(m.Artist()).To(Equal("Artist"))
+		Expect(m.AlbumArtist()).To(Equal("Album Artist"))
+		Expect(m.Composer()).To(Equal("Composer"))
+		Expect(m.Compilation()).To(BeFalse())
+		Expect(m.Genre()).To(Equal("Rock"))
+		Expect(m.Year()).To(Equal(2014))
+		n, t := m.TrackNumber()
+		Expect(n).To(Equal(2))
+		Expect(t).To(Equal(10))
+		n, t = m.DiscNumber()
+		Expect(n).To(Equal(1))
+		Expect(t).To(Equal(2))
+		Expect(m.HasPicture()).To(BeTrue())
+		Expect(m.Duration()).To(Equal(1))
+		Expect(m.BitRate()).To(Equal(476))
+		Expect(m.FilePath()).To(Equal("../tests/fixtures/test.mp3"))
+		Expect(m.Suffix()).To(Equal("mp3"))
+		Expect(m.Size()).To(Equal(60845))
+	})
+
+	It("correctly parses ogg file with no tags", func() {
+		m, err := ExtractMetadata("../tests/fixtures/test.ogg")
+		Expect(err).To(BeNil())
+		Expect(m.Title()).To(BeEmpty())
+		Expect(m.HasPicture()).To(BeFalse())
+		Expect(m.Duration()).To(Equal(3))
+		Expect(m.BitRate()).To(Equal(9))
+		Expect(m.Suffix()).To(Equal("ogg"))
+		Expect(m.FilePath()).To(Equal("../tests/fixtures/test.ogg"))
+		Expect(m.Size()).To(Equal(4408))
+	})
+
+	It("returns error for invalid media file", func() {
+		_, err := ExtractMetadata("../tests/fixtures/itunes-library.xml")
+		Expect(err).ToNot(BeNil())
+	})
+
+	It("returns error for file not found", func() {
+		_, err := ExtractMetadata("../tests/fixtures/NOT-FOUND.mp3")
+		Expect(err).ToNot(BeNil())
+	})
+})

scanner/scanner.go

@@ -0,0 +1,121 @@
+package scanner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/cloudsonic/sonic-server/log"
+	"github.com/cloudsonic/sonic-server/model"
+)
+
+type Scanner struct {
+	folders map[string]FolderScanner
+	repos   Repositories
+}
+
+type Repositories struct {
+	folder    model.MediaFolderRepository
+	mediaFile model.MediaFileRepository
+	album     model.AlbumRepository
+	artist    model.ArtistRepository
+	index     model.ArtistIndexRepository
+	playlist  model.PlaylistRepository
+	property  model.PropertyRepository
+}
+
+func New(mfRepo model.MediaFileRepository, albumRepo model.AlbumRepository, artistRepo model.ArtistRepository, idxRepo model.ArtistIndexRepository, plsRepo model.PlaylistRepository, folderRepo model.MediaFolderRepository, property model.PropertyRepository) *Scanner {
+	repos := Repositories{
+		folder:    folderRepo,
+		mediaFile: mfRepo,
+		album:     albumRepo,
+		artist:    artistRepo,
+		index:     idxRepo,
+		playlist:  plsRepo,
+		property:  property,
+	}
+	s := &Scanner{repos: repos, folders: map[string]FolderScanner{}}
+	s.loadFolders()
+	return s
+}
+
+func (s *Scanner) Rescan(mediaFolder string, fullRescan bool) error {
+	folderScanner := s.folders[mediaFolder]
+	start := time.Now()
+
+	lastModifiedSince := time.Time{}
+	if !fullRescan {
+		lastModifiedSince = s.getLastModifiedSince(mediaFolder)
+		log.Debug("Scanning folder", "folder", mediaFolder, "lastModifiedSince", lastModifiedSince)
+	} else {
+		log.Debug("Scanning folder (full scan)", "folder", mediaFolder)
+	}
+
+	err := folderScanner.Scan(nil, lastModifiedSince)
+	if err != nil {
+		log.Error("Error importing MediaFolder", "folder", mediaFolder, err)
+	}
+
+	s.updateLastModifiedSince(mediaFolder, start)
+	log.Debug("Finished scanning folder", "folder", mediaFolder, "elapsed", time.Since(start))
+	return err
+}
+
+func (s *Scanner) RescanAll(fullRescan bool) error {
+	var hasError bool
+	for folder := range s.folders {
+		err := s.Rescan(folder, fullRescan)
+		hasError = hasError || err != nil
+	}
+	if hasError {
+		log.Error("Errors while scanning media. Please check the logs")
+		return errors.New("errors while scanning media")
+	}
+	return nil
+}
+
+func (s *Scanner) Status() []StatusInfo { return nil }
+
+func (i *Scanner) getLastModifiedSince(folder string) time.Time {
+	ms, err := i.repos.property.Get(model.PropLastScan + "-" + folder)
+	if err != nil {
+		return time.Time{}
+	}
+	if ms == "" {
+		return time.Time{}
+	}
+	s, _ := strconv.ParseInt(ms, 10, 64)
+	return time.Unix(0, s*int64(time.Millisecond))
+}
+
+func (s *Scanner) updateLastModifiedSince(folder string, t time.Time) {
+	millis := t.UnixNano() / int64(time.Millisecond)
+	s.repos.property.Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis))
+}
+
+func (s *Scanner) loadFolders() {
+	fs, _ := s.repos.folder.GetAll()
+	for _, f := range fs {
+		log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
+		s.folders[f.Path] = NewTagScanner(f.Path, s.repos)
+	}
+}
+
+type Status int
+
+const (
+	StatusComplete Status = iota
+	StatusInProgress
+	StatusError
+)
+
+type StatusInfo struct {
+	MediaFolder string
+	Status      Status
+}
+
+type FolderScanner interface {
+	Scan(ctx context.Context, lastModifiedSince time.Time) error
+}

scanner/scanner_suite_test.go

@@ -0,0 +1,36 @@
+package scanner
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cloudsonic/sonic-server/conf"
+	"github.com/cloudsonic/sonic-server/log"
+	"github.com/cloudsonic/sonic-server/persistence"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestScanner(t *testing.T) {
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Scanner Suite")
+}
+
+var _ = XDescribe("TODO: REMOVE", func() {
+	conf.Sonic.DbPath = "./testDB"
+	log.SetLevel(log.LevelDebug)
+	repos := Repositories{
+		folder:    persistence.NewMediaFolderRepository(),
+		mediaFile: persistence.NewMediaFileRepository(),
+		album:     persistence.NewAlbumRepository(),
+		artist:    persistence.NewArtistRepository(),
+		index:     persistence.NewArtistIndexRepository(),
+		playlist:  nil,
+	}
+	It("WORKS!", func() {
+		t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", repos)
+		//t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", repos)
+		Expect(t.Scan(nil, time.Time{})).To(BeNil())
+	})
+})

scanner/tag_scanner.go

@@ -0,0 +1,273 @@
+package scanner
+
+import (
+	"context"
+	"crypto/md5"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudsonic/sonic-server/log"
+	"github.com/cloudsonic/sonic-server/model"
+)
+
+type TagScanner struct {
+	rootFolder string
+	repos      Repositories
+}
+
+func NewTagScanner(rootFolder string, repos Repositories) *TagScanner {
+	return &TagScanner{
+		rootFolder: rootFolder,
+		repos:      repos,
+	}
+}
+
+// Scan algorithm overview:
+// For each changed: Get all files from DB that starts with it, scan each file:
+//		if changed or new, delete from DB and add new from the file
+// 		if not found, delete from DB
+//		scan and add the new ones
+// For each deleted: delete all files from DB that starts with it
+// Create new albums/artists, update counters (how?)
+//      collect all albumids and artistids from previous steps
+//      run something like this (for albums):
+//          select album_id, album, f.artist, f.compilation, max(f.year), count(*), sum(f.play_count), max(f.updated_at), a.id from media_file f left outer join album a on f.album_id = a.id group by album_id;
+//      when a.id is not null update, else  insert (collect all inserts and run just one InsertMulti)
+// Delete all empty albums, delete all empty Artists
+// Recreate ArtistIndex
+func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
+	detector := NewChangeDetector(s.rootFolder, lastModifiedSince)
+	changed, deleted, err := detector.Scan()
+	if err != nil {
+		return err
+	}
+
+	if len(changed)+len(deleted) == 0 {
+		return nil
+	}
+
+	log.Info("Folder changes found", "changed", len(changed), "deleted", len(deleted))
+
+	updatedArtists := map[string]bool{}
+	updatedAlbums := map[string]bool{}
+
+	for _, c := range changed {
+		err := s.processChangedDir(c, updatedArtists, updatedAlbums)
+		if err != nil {
+			return err
+		}
+	}
+	for _, c := range deleted {
+		err := s.processDeletedDir(c, updatedArtists, updatedAlbums)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = s.refreshAlbums(updatedAlbums)
+	if err != nil {
+		return err
+	}
+
+	err = s.refreshArtists(updatedArtists)
+	if err != nil {
+		return err
+	}
+
+	err = s.repos.index.Refresh()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *TagScanner) refreshAlbums(updatedAlbums map[string]bool) error {
+	var ids []string
+	for id := range updatedAlbums {
+		ids = append(ids, id)
+	}
+	return s.repos.album.Refresh(ids...)
+}
+
+func (s *TagScanner) refreshArtists(updatedArtists map[string]bool) error {
+	var ids []string
+	for id := range updatedArtists {
+		ids = append(ids, id)
+	}
+	return s.repos.artist.Refresh(ids...)
+}
+
+func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
+	dir = path.Join(s.rootFolder, dir)
+
+	start := time.Now()
+
+	// Load folder's current tracks from DB into a map
+	currentTracks := map[string]model.MediaFile{}
+	ct, err := s.repos.mediaFile.FindByPath(dir)
+	if err != nil {
+		return err
+	}
+	for _, t := range ct {
+		currentTracks[t.ID] = t
+		updatedArtists[t.ArtistID] = true
+		updatedAlbums[t.AlbumID] = true
+	}
+
+	// Load tracks from the folder
+	newTracks, err := s.loadTracks(dir)
+	if err != nil {
+		return err
+	}
+
+	// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
+	log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(newTracks))
+	numUpdatedTracks := 0
+	numPurgedTracks := 0
+	for _, n := range newTracks {
+		c, ok := currentTracks[n.ID]
+		if !ok || (ok && n.UpdatedAt.After(c.UpdatedAt)) {
+			err := s.repos.mediaFile.Put(&n)
+			updatedArtists[n.ArtistID] = true
+			updatedAlbums[n.AlbumID] = true
+			numUpdatedTracks++
+			if err != nil {
+				return err
+			}
+		}
+		delete(currentTracks, n.ID)
+	}
+
+	// Remaining tracks from DB that are not in the folder are deleted
+	for id := range currentTracks {
+		numPurgedTracks++
+		if err := s.repos.mediaFile.Delete(id); err != nil {
+			return err
+		}
+	}
+
+	log.Debug("Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start))
+	return nil
+}
+
+func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
+	dir = path.Join(s.rootFolder, dir)
+
+	ct, err := s.repos.mediaFile.FindByPath(dir)
+	if err != nil {
+		return err
+	}
+	for _, t := range ct {
+		updatedArtists[t.ArtistID] = true
+		updatedAlbums[t.AlbumID] = true
+	}
+
+	return s.repos.mediaFile.DeleteByPath(dir)
+}
+
+func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
+	dir, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+	var mds model.MediaFiles
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		filePath := path.Join(dirPath, f.Name())
+		md, err := ExtractMetadata(filePath)
+		if err != nil {
+			continue
+		}
+		mf := s.toMediaFile(md)
+		mds = append(mds, mf)
+	}
+	return mds, nil
+}
+
+func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
+	mf := model.MediaFile{}
+	mf.ID = s.trackID(md)
+	mf.Title = s.mapTrackTitle(md)
+	mf.Album = md.Album()
+	mf.AlbumID = s.albumID(md)
+	mf.Album = s.mapAlbumName(md)
+	if md.Artist() == "" {
+		mf.Artist = "[Unknown Artist]"
+	} else {
+		mf.Artist = md.Artist()
+	}
+	mf.ArtistID = s.artistID(md)
+	mf.AlbumArtist = md.AlbumArtist()
+	mf.Genre = md.Genre()
+	mf.Compilation = md.Compilation()
+	mf.Year = md.Year()
+	mf.TrackNumber, _ = md.TrackNumber()
+	mf.DiscNumber, _ = md.DiscNumber()
+	mf.Duration = md.Duration()
+	mf.BitRate = md.BitRate()
+	mf.Path = md.FilePath()
+	mf.Suffix = md.Suffix()
+	mf.Size = strconv.Itoa(md.Size())
+	mf.HasCoverArt = md.HasPicture()
+
+	// TODO Get Creation time. https://github.com/djherbis/times ?
+	mf.CreatedAt = md.ModificationTime()
+	mf.UpdatedAt = md.ModificationTime()
+
+	return mf
+}
+
+func (s *TagScanner) mapTrackTitle(md *Metadata) string {
+	if md.Title() == "" {
+		s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
+		e := filepath.Ext(s)
+		return strings.TrimSuffix(s, e)
+	}
+	return md.Title()
+}
+
+func (s *TagScanner) mapArtistName(md *Metadata) string {
+	switch {
+	case md.Compilation():
+		return "Various Artists"
+	case md.AlbumArtist() != "":
+		return md.AlbumArtist()
+	case md.Artist() != "":
+		return md.Artist()
+	default:
+		return "[Unknown Artist]"
+	}
+}
+
+func (s *TagScanner) mapAlbumName(md *Metadata) string {
+	name := md.Album()
+	if name == "" {
+		return "[Unknown Album]"
+	}
+	return name
+}
+
+func (s *TagScanner) trackID(md *Metadata) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
+}
+
+func (s *TagScanner) albumID(md *Metadata) string {
+	albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapArtistName(md), s.mapAlbumName(md)))
+	return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
+}
+
+func (s *TagScanner) artistID(md *Metadata) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
+}

server/app.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cloudsonic/sonic-server/conf"
 	"github.com/cloudsonic/sonic-server/log"
+	"github.com/cloudsonic/sonic-server/scanner"
 	"github.com/cloudsonic/sonic-server/scanner_legacy"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
@@ -19,17 +20,24 @@ const Version = "0.2"
 
 type Server struct {
 	Importer *scanner_legacy.Importer
+	Scanner  *scanner.Scanner
 	router   *chi.Mux
 }
 
-func New(importer *scanner_legacy.Importer) *Server {
+func New(importer *scanner_legacy.Importer, scanner *scanner.Scanner) *Server {
-	a := &Server{Importer: importer}
+	a := &Server{Importer: importer, Scanner: scanner}
 	if !conf.Sonic.DevDisableBanner {
 		showBanner(Version)
 	}
 	initMimeTypes()
 	a.initRoutes()
-	a.initImporter()
+	if conf.Sonic.DevUseFileScanner {
+		log.Info("Using Folder Scanner", "folder", conf.Sonic.MusicFolder)
+		a.initScanner()
+	} else {
+		log.Info("Using iTunes Importer", "xml", conf.Sonic.MusicFolder)
+		a.initImporter()
+	}
 	return a
 }
 
@@ -67,22 +75,34 @@ func (a *Server) initRoutes() {
 	a.router = r
 }
 
-func (a *Server) initImporter() {
+func (a *Server) initScanner() {
-	go a.startPeriodicScans()
+	go func() {
+		for {
+			select {
+			case <-time.After(5 * time.Second):
+				err := a.Scanner.RescanAll(false)
+				if err != nil {
+					log.Error("Error scanning media folder", "folder", conf.Sonic.MusicFolder, err)
+				}
+			}
+		}
+	}()
 }
 
-func (a *Server) startPeriodicScans() {
+func (a *Server) initImporter() {
-	first := true
+	go func() {
-	for {
+		first := true
-		select {
+		for {
-		case <-time.After(5 * time.Second):
+			select {
-			if first {
+			case <-time.After(5 * time.Second):
-				log.Info("Started iTunes scanner", "xml", conf.Sonic.MusicFolder)
+				if first {
-				first = false
+					log.Info("Started iTunes scanner", "xml", conf.Sonic.MusicFolder)
+					first = false
+				}
+				a.Importer.CheckForUpdates(false)
 			}
-			a.Importer.CheckForUpdates(false)
 		}
-	}
+	}()
 }
 
 func FileServer(r chi.Router, path string, root http.FileSystem) {

wire_gen.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cloudsonic/sonic-server/engine"
 	"github.com/cloudsonic/sonic-server/itunesbridge"
 	"github.com/cloudsonic/sonic-server/persistence"
+	"github.com/cloudsonic/sonic-server/scanner"
 	"github.com/cloudsonic/sonic-server/scanner_legacy"
 	"github.com/cloudsonic/sonic-server/server"
 	"github.com/google/wire"
@@ -27,7 +28,9 @@ func CreateApp(musicFolder string) *server.Server {
 	playlistRepository := persistence.NewPlaylistRepository()
 	propertyRepository := persistence.NewPropertyRepository()
 	importer := scanner_legacy.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, propertyRepository)
-	serverServer := server.New(importer)
+	mediaFolderRepository := persistence.NewMediaFolderRepository()
+	scannerScanner := scanner.New(mediaFileRepository, albumRepository, artistRepository, artistIndexRepository, playlistRepository, mediaFolderRepository, propertyRepository)
+	serverServer := server.New(importer, scannerScanner)
 	return serverServer
 }
 
@@ -55,4 +58,4 @@ func CreateSubsonicAPIRouter() *api.Router {
 
 // wire_injectors.go:
 
-var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner_legacy.Set, api.NewRouter, persistence.Set)
+var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner_legacy.Set, scanner.New, api.NewRouter, persistence.Set)

wire_injectors.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cloudsonic/sonic-server/engine"
 	"github.com/cloudsonic/sonic-server/itunesbridge"
 	"github.com/cloudsonic/sonic-server/persistence"
+	"github.com/cloudsonic/sonic-server/scanner"
 	"github.com/cloudsonic/sonic-server/scanner_legacy"
 	"github.com/cloudsonic/sonic-server/server"
 	"github.com/google/wire"
@@ -16,6 +17,7 @@ var allProviders = wire.NewSet(
 	itunesbridge.NewItunesControl,
 	engine.Set,
 	scanner_legacy.Set,
+	scanner.New,
 	api.NewRouter,
 	persistence.Set,
 )