Extract metadata from files in a directory in batch (one call to ffmpeg for all files)

conf/configuration.go

@@ -17,16 +17,16 @@ type sonic struct {
 
 	DisableDownsampling bool   `default:"false"`
 	DownsampleCommand   string `default:"ffmpeg -i %s -map 0:0 -b:a %bk -v 0 -f mp3 -"`
-	//ProbeCommand        string `default:"ffprobe -v quiet -print_format json -show_format %s"`
+	ProbeCommand        string `default:"ffmpeg %s -f ffmetadata"`
-	ProbeCommand       string `default:"ffmpeg -i %s -f ffmetadata"`
+	PlsIgnoreFolders    bool   `default:"true"`
-	PlsIgnoreFolders   bool   `default:"true"`
+	PlsIgnoredPatterns  string `default:"^iCloud;\\~"`
-	PlsIgnoredPatterns string `default:"^iCloud;\\~"`
 
 	// DevFlags
 	LogLevel                 string `default:"info"`
 	DevDisableAuthentication bool   `default:"false"`
 	DevDisableFileCheck      bool   `default:"false"`
 	DevDisableBanner         bool   `default:"false"`
+	DevInitialPassword       string `default:""`
 }
 
 var Sonic *sonic

scanner/metadata_ffmpeg.go

@@ -2,7 +2,6 @@ package scanner
 
 import (
 	"bufio"
-	"bytes"
 	"errors"
 	"mime"
 	"os"
@@ -24,31 +23,6 @@ type Metadata struct {
 	tags     map[string]string
 }
 
-func ExtractMetadata(filePath string) (*Metadata, error) {
-	m := &Metadata{filePath: filePath, tags: map[string]string{}}
-	extension := path.Ext(filePath)
-	if !isAudioFile(extension) {
-		return nil, errors.New("not an audio file")
-	}
-	m.suffix = strings.ToLower(strings.TrimPrefix(extension, "."))
-	fi, err := os.Stat(filePath)
-	if err != nil {
-		return nil, err
-	}
-	m.fileInfo = fi
-
-	err = m.probe(filePath)
-	if len(m.tags) == 0 {
-		return nil, errors.New("not a media file")
-	}
-	return m, err
-}
-
-func isAudioFile(extension string) bool {
-	typ := mime.TypeByExtension(extension)
-	return strings.HasPrefix(typ, "audio/")
-}
-
 func (m *Metadata) Title() string               { return m.tags["title"] }
 func (m *Metadata) Album() string               { return m.tags["album"] }
 func (m *Metadata) Artist() string              { return m.tags["artist"] }
@@ -67,40 +41,116 @@ func (m *Metadata) FilePath() string            { return m.filePath }
 func (m *Metadata) Suffix() string              { return m.suffix }
 func (m *Metadata) Size() int                   { return int(m.fileInfo.Size()) }
 
-func (m *Metadata) probe(filePath string) error {
+func ExtractAllMetadata(dirPath string) (map[string]*Metadata, error) {
-	cmdLine, args := createProbeCommand(filePath)
+	dir, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+	var audioFiles []string
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		filePath := path.Join(dirPath, f.Name())
+		extension := path.Ext(filePath)
+		if !isAudioFile(extension) {
+			continue
+		}
+		audioFiles = append(audioFiles, filePath)
+	}
+
+	if len(audioFiles) == 0 {
+		return map[string]*Metadata{}, nil
+	}
+	return probe(audioFiles)
+}
+
+func probe(inputs []string) (map[string]*Metadata, error) {
+	cmdLine, args := createProbeCommand(inputs)
 
 	log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
 	cmd := exec.Command(cmdLine, args...)
 	output, _ := cmd.CombinedOutput()
-	if len(output) == 0 || bytes.Contains(output, []byte("No such file or directory")) {
+	mds := map[string]*Metadata{}
-		return errors.New("error extracting metadata from " + filePath)
+	if len(output) == 0 {
+		return mds, errors.New("error extracting metadata files")
 	}
-	return m.parseOutput(output)
+	infos := parseOutput(string(output))
+	for file, info := range infos {
+		md, err := extractMetadata(file, info)
+		if err == nil {
+			mds[file] = md
+		}
+	}
+	return mds, nil
+}
+
+var inputRegex = regexp.MustCompile(`(?m)^Input #\d+,.*,\sfrom\s'(.*)'`)
+
+func parseOutput(output string) map[string]string {
+	split := map[string]string{}
+	all := inputRegex.FindAllStringSubmatchIndex(string(output), -1)
+	for i, loc := range all {
+		// Filename is the first captured group
+		file := output[loc[2]:loc[3]]
+
+		// File info is everything from the match, up until the beginning of the next match
+		info := ""
+		initial := loc[1]
+		if i < len(all)-1 {
+			end := all[i+1][0] - 1
+			info = output[initial:end]
+		} else {
+			// if this is the last match
+			info = output[initial:]
+		}
+		split[file] = info
+	}
+	return split
+}
+
+func extractMetadata(filePath, info string) (*Metadata, error) {
+	m := &Metadata{filePath: filePath, tags: map[string]string{}}
+	m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
+	m.parseInfo(info)
+	m.fileInfo, _ = os.Stat(filePath)
+	if len(m.tags) == 0 {
+		return nil, errors.New("not a media file")
+	}
+	return m, nil
+}
+
+func isAudioFile(extension string) bool {
+	typ := mime.TypeByExtension(extension)
+	return strings.HasPrefix(typ, "audio/")
 }
 
 var (
 	tagsRx = map[*regexp.Regexp]string{
-		regexp.MustCompile(`^\s+compilation\s+:(.*)`):     "compilation",
+		regexp.MustCompile(`^\s+compilation\s+:(.*)`):      "compilation",
-		regexp.MustCompile(`^\s+genre\s+:\s(.*)`):         "genre",
+		regexp.MustCompile(`^\s+genre\s+:\s(.*)`):          "genre",
-		regexp.MustCompile(`^\s+title\s+:\s(.*)`):         "title",
+		regexp.MustCompile(`^\s+title\s+:\s(.*)`):          "title",
-		regexp.MustCompile(`^\s{4}comment\s+:\s(.*)`):     "comment",
+		regexp.MustCompile(`^\s{4}comment\s+:\s(.*)`):      "comment",
-		regexp.MustCompile(`^\s+artist\s+:\s(.*)`):        "artist",
+		regexp.MustCompile(`^\s+artist\s+:\s(.*)`):         "artist",
-		regexp.MustCompile(`^\s+album_artist\s+:\s(.*)`):  "album_artist",
+		regexp.MustCompile(`^\s+album_artist\s+:\s(.*)`):   "album_artist",
-		regexp.MustCompile(`^\s+TCM\s+:\s(.*)`):           "composer",
+		regexp.MustCompile(`^\s+TCM\s+:\s(.*)`):            "composer",
-		regexp.MustCompile(`^\s+album\s+:\s(.*)`):         "album",
+		regexp.MustCompile(`^\s+album\s+:\s(.*)`):          "album",
-		regexp.MustCompile(`^\s+track\s+:\s(.*)`):         "trackNum",
+		regexp.MustCompile(`^\s+track\s+:\s(.*)`):          "trackNum",
-		regexp.MustCompile(`^\s+disc\s+:\s(.*)`):          "discNum",
+		regexp.MustCompile(`^\s+disc\s+:\s(.*)`):           "discNum",
-		regexp.MustCompile(`^\s+TPA\s+:\s(.*)`):           "discNum",
+		regexp.MustCompile(`^\s+TPA\s+:\s(.*)`):            "discNum",
-		regexp.MustCompile(`^\s+date\s+:\s(.*)`):          "year",
+		regexp.MustCompile(`^\s+date\s+:\s(.*)`):           "year",
-		regexp.MustCompile(`^\s{4}Stream #0:1: (.+)\:\s`): "hasPicture",
+		regexp.MustCompile(`^\s{4}Stream #\d+:1: (.+):\s`): "hasPicture",
 	}
 
 	durationRx = regexp.MustCompile(`^\s\sDuration: ([\d.:]+).*bitrate: (\d+)`)
 )
 
-func (m *Metadata) parseOutput(output []byte) error {
+func (m *Metadata) parseInfo(info string) {
-	reader := strings.NewReader(string(output))
+	reader := strings.NewReader(info)
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		line := scanner.Text()
@@ -120,7 +170,6 @@ func (m *Metadata) parseOutput(output []byte) error {
 			}
 		}
 	}
-	return nil
 }
 
 func (m *Metadata) parseInt(tagName string) int {
@@ -165,14 +214,20 @@ func (m *Metadata) parseDuration(tagName string) int {
 	return 0
 }
 
-func createProbeCommand(filePath string) (string, []string) {
+func createProbeCommand(inputs []string) (string, []string) {
 	cmd := conf.Sonic.ProbeCommand
 
 	split := strings.Split(cmd, " ")
-	for i, s := range split {
+	args := make([]string, 0)
-		s = strings.Replace(s, "%s", filePath, -1)
+	for _, s := range split {
-		split[i] = s
+		if s == "%s" {
+			for _, inp := range inputs {
+				args = append(args, "-i", inp)
+			}
+			continue
+		}
+		args = append(args, s)
 	}
 
-	return split[0], split[1:]
+	return args[0], args[1:]
 }

scanner/metadata_ffprobe.go

@@ -1,142 +0,0 @@
-//+build ignored
-
-package scanner
-
-import (
-	"encoding/json"
-	"errors"
-	"os"
-	"os/exec"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/cloudsonic/sonic-server/conf"
-	"github.com/cloudsonic/sonic-server/log"
-	"github.com/dhowden/tag"
-)
-
-type Metadata struct {
-	filePath    string
-	suffix      string
-	fileInfo    os.FileInfo
-	t           tag.Metadata
-	duration    int
-	bitRate     int
-	compilation bool
-}
-
-func ExtractMetadata(filePath string) (*Metadata, error) {
-	m := &Metadata{filePath: filePath}
-	m.suffix = strings.ToLower(strings.TrimPrefix(path.Ext(filePath), "."))
-	fi, err := os.Stat(filePath)
-	if err != nil {
-		return nil, err
-	}
-	m.fileInfo = fi
-
-	f, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	t, err := tag.ReadFrom(f)
-	if err != nil {
-		return nil, err
-	}
-	m.t = t
-
-	err = m.probe(filePath)
-	return m, err
-}
-
-func (m *Metadata) Title() string               { return m.t.Title() }
-func (m *Metadata) Album() string               { return m.t.Album() }
-func (m *Metadata) Artist() string              { return m.t.Artist() }
-func (m *Metadata) AlbumArtist() string         { return m.t.AlbumArtist() }
-func (m *Metadata) Composer() string            { return m.t.Composer() }
-func (m *Metadata) Genre() string               { return m.t.Genre() }
-func (m *Metadata) Year() int                   { return m.t.Year() }
-func (m *Metadata) TrackNumber() (int, int)     { return m.t.Track() }
-func (m *Metadata) DiscNumber() (int, int)      { return m.t.Disc() }
-func (m *Metadata) HasPicture() bool            { return m.t.Picture() != nil }
-func (m *Metadata) Compilation() bool           { return m.compilation }
-func (m *Metadata) Duration() int               { return m.duration }
-func (m *Metadata) BitRate() int                { return m.bitRate }
-func (m *Metadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
-func (m *Metadata) FilePath() string            { return m.filePath }
-func (m *Metadata) Suffix() string              { return m.suffix }
-func (m *Metadata) Size() int                   { return int(m.fileInfo.Size()) }
-
-// probe analyzes the file and returns duration in seconds and bitRate in kb/s.
-// It uses the ffprobe external tool, configured in conf.Sonic.ProbeCommand
-func (m *Metadata) probe(filePath string) error {
-	cmdLine, args := createProbeCommand(filePath)
-
-	log.Trace("Executing command", "cmdLine", cmdLine, "args", args)
-	cmd := exec.Command(cmdLine, args...)
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		return err
-	}
-	return m.parseOutput(output)
-}
-
-func (m *Metadata) parseInt(objItf interface{}, field string) (int, error) {
-	obj := objItf.(map[string]interface{})
-	s, ok := obj[field].(string)
-	if !ok {
-		return -1, errors.New("invalid ffprobe output field obj." + field)
-	}
-	fDuration, err := strconv.ParseFloat(s, 64)
-	if err != nil {
-		return -1, err
-	}
-	return int(fDuration), nil
-}
-
-func (m *Metadata) parseOutput(output []byte) error {
-	var data map[string]map[string]interface{}
-	err := json.Unmarshal(output, &data)
-	if err != nil {
-		return err
-	}
-
-	format, ok := data["format"]
-	if !ok {
-		err = errors.New("invalid ffprobe output. no format found")
-		return err
-	}
-
-	if tags, ok := format["tags"]; ok {
-		c, _ := m.parseInt(tags, "compilation")
-		m.compilation = c == 1
-	}
-
-	m.duration, err = m.parseInt(format, "duration")
-	if err != nil {
-		return err
-	}
-
-	m.bitRate, err = m.parseInt(format, "bit_rate")
-	m.bitRate = m.bitRate / 1000
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func createProbeCommand(filePath string) (string, []string) {
-	cmd := conf.Sonic.ProbeCommand
-
-	split := strings.Split(cmd, " ")
-	for i, s := range split {
-		s = strings.Replace(s, "%s", filePath, -1)
-		split[i] = s
-	}
-
-	return split[0], split[1:]
-}

scanner/metadata_test.go

@@ -6,9 +6,12 @@ import (
 )
 
 var _ = Describe("Metadata", func() {
-	FIt("correctly parses mp3 file", func() {
+	It("correctly parses metadata from all files in folder", func() {
-		m, err := ExtractMetadata("../tests/fixtures/test.mp3")
+		mds, err := ExtractAllMetadata("../tests/fixtures")
-		Expect(err).To(BeNil())
+		Expect(err).NotTo(HaveOccurred())
+		Expect(mds).To(HaveLen(3))
+
+		m := mds["../tests/fixtures/test.mp3"]
 		Expect(m.Title()).To(Equal("Song"))
 		Expect(m.Album()).To(Equal("Album"))
 		Expect(m.Artist()).To(Equal("Artist"))
@@ -29,10 +32,8 @@ var _ = Describe("Metadata", func() {
 		Expect(m.FilePath()).To(Equal("../tests/fixtures/test.mp3"))
 		Expect(m.Suffix()).To(Equal("mp3"))
 		Expect(m.Size()).To(Equal(60845))
-	})
 
-	It("correctly parses ogg file with no tags", func() {
+		m = mds["../tests/fixtures/test.ogg"]
-		m, err := ExtractMetadata("../tests/fixtures/test.ogg")
 		Expect(err).To(BeNil())
 		Expect(m.Title()).To(BeEmpty())
 		Expect(m.HasPicture()).To(BeFalse())
@@ -43,13 +44,12 @@ var _ = Describe("Metadata", func() {
 		Expect(m.Size()).To(Equal(4408))
 	})
 
-	FIt("returns error for invalid media file", func() {
+	It("returns error if path does not exist", func() {
-		_, err := ExtractMetadata("../tests/fixtures/itunes-library.xml")
+		_, err := ExtractAllMetadata("./INVALID/PATH")
-		Expect(err).ToNot(BeNil())
+		Expect(err).To(HaveOccurred())
 	})
 
-	It("returns error for file not found", func() {
+	It("returns empty map if there are no audio files in path", func() {
-		_, err := ExtractMetadata("../tests/fixtures/NOT-FOUND.mp3")
+		Expect(ExtractAllMetadata(".")).To(BeEmpty())
-		Expect(err).ToNot(BeNil())
 	})
 })

scanner/scanner_suite_test.go

@@ -19,10 +19,11 @@ func xTestScanner(t *testing.T) {
 }
 
 var _ = XDescribe("TODO: REMOVE", func() {
-	conf.Sonic.DbPath = "./testDB"
-	log.SetLevel(log.LevelDebug)
-	ds := persistence.New()
 	It("WORKS!", func() {
+		conf.Sonic.DbPath = "./testDB"
+		log.SetLevel(log.LevelDebug)
+		ds := persistence.New()
+
 		t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", ds)
 		//t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", ds)
 		Expect(t.Scan(nil, time.Time{})).To(BeNil())

scanner/tag_scanner.go

@@ -208,28 +208,16 @@ func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]boo
 }
 
 func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
-	dir, err := os.Open(dirPath)
+	mds, err := ExtractAllMetadata(dirPath)
 	if err != nil {
 		return nil, err
 	}
-	files, err := dir.Readdir(-1)
+	var mfs model.MediaFiles
-	if err != nil {
+	for _, md := range mds {
-		return nil, err
-	}
-	var mds model.MediaFiles
-	for _, f := range files {
-		if f.IsDir() {
-			continue
-		}
-		filePath := path.Join(dirPath, f.Name())
-		md, err := ExtractMetadata(filePath)
-		if err != nil {
-			continue
-		}
 		mf := s.toMediaFile(md)
-		mds = append(mds, mf)
+		mfs = append(mfs, mf)
 	}
-	return mds, nil
+	return mfs, nil
 }
 
 func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {

server/initial_setup.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/cloudsonic/sonic-server/conf"
 	"github.com/cloudsonic/sonic-server/consts"
 	"github.com/cloudsonic/sonic-server/log"
 	"github.com/cloudsonic/sonic-server/model"
@@ -50,14 +51,18 @@ func createDefaultUser(ds model.DataStore) error {
 	}
 	if c == 0 {
 		id, _ := uuid.NewRandom()
-		initialPassword, _ := uuid.NewRandom()
+		random, _ := uuid.NewRandom()
+		initialPassword := random.String()
+		if conf.Sonic.DevInitialPassword != "" {
+			initialPassword = conf.Sonic.DevInitialPassword
+		}
 		log.Warn("Creating initial user. Please change the password!", "user", consts.InitialUserName, "password", initialPassword)
 		initialUser := model.User{
 			ID:       id.String(),
 			UserName: consts.InitialUserName,
 			Name:     consts.InitialName,
 			Email:    "",
-			Password: initialPassword.String(),
+			Password: initialPassword,
 			IsAdmin:  true,
 		}
 		err := ds.User().Put(&initialUser)