From cf8d08ec26e40b64c89a8bedb7333bc6f14059f0 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 8 Oct 2021 22:23:43 -0400 Subject: [PATCH] Initial drafts for Smart Playlists --- .../20211008205505_add_smart_playlist.go | 37 +++ model/model_suite_test.go | 18 ++ model/playlist.go | 4 + model/smartplaylist.go | 106 +++++++ model/smartplaylist_test.go | 101 +++++++ persistence/sql_smartplaylist.go | 273 ++++++++++++++++++ persistence/sql_smartplaylist_test.go | 102 +++++++ 7 files changed, 641 insertions(+) create mode 100644 db/migration/20211008205505_add_smart_playlist.go create mode 100644 model/model_suite_test.go create mode 100644 model/smartplaylist.go create mode 100644 model/smartplaylist_test.go create mode 100644 persistence/sql_smartplaylist.go create mode 100644 persistence/sql_smartplaylist_test.go diff --git a/db/migration/20211008205505_add_smart_playlist.go b/db/migration/20211008205505_add_smart_playlist.go new file mode 100644 index 000000000..b8c2a96e4 --- /dev/null +++ b/db/migration/20211008205505_add_smart_playlist.go @@ -0,0 +1,37 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(upAddSmartPlaylist, downAddSmartPlaylist) +} + +func upAddSmartPlaylist(tx *sql.Tx) error { + _, err := tx.Exec(` +alter table playlist + add column rules varchar null; +alter table playlist + add column evaluated_at datetime null; +create index if not exists playlist_evaluated_at + on playlist(evaluated_at); + +create table playlist_fields ( + field varchar(255) not null, + playlist_id varchar(255) not null + constraint playlist_fields_playlist_id_fk + references playlist + on update cascade on delete cascade +); +create unique index playlist_fields_idx + on playlist_fields (field, playlist_id); +`) + return err +} + +func downAddSmartPlaylist(tx *sql.Tx) error { + return nil +} diff --git a/model/model_suite_test.go b/model/model_suite_test.go new file mode 100644 index 000000000..bc2658e14 --- /dev/null +++ b/model/model_suite_test.go @@ -0,0 +1,18 @@ +package model_test + +import ( + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestModel(t *testing.T) { + tests.Init(t, true) + log.SetLevel(log.LevelCritical) + RegisterFailHandler(Fail) + RunSpecs(t, "Model Suite") +} diff --git a/model/playlist.go b/model/playlist.go index 87a60ab9b..75552b535 100644 --- a/model/playlist.go +++ b/model/playlist.go @@ -18,6 +18,10 @@ type Playlist struct { Sync bool `structs:"sync" json:"sync"` CreatedAt time.Time `structs:"created_at" json:"createdAt"` UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` + + // SmartPlaylist attributes + //Rules *SmartPlaylist `structs:"rules" json:"rules"` + //EvaluatedAt time.Time `structs:"evaluated_at" json:"evaluatedAt"` } type Playlists []Playlist diff --git a/model/smartplaylist.go b/model/smartplaylist.go new file mode 100644 index 000000000..70b6d1ca7 --- /dev/null +++ b/model/smartplaylist.go @@ -0,0 +1,106 @@ +package model + +import ( + "encoding/json" + "errors" +) + +type SmartPlaylist struct { + RuleGroup + Order string `json:"order,omitempty"` + Limit int `json:"limit,omitempty"` +} + +type RuleGroup struct { + Combinator string `json:"combinator"` + Rules Rules `json:"rules"` +} + +type Rules []IRule + +type IRule interface { + Fields() []string +} + +type Rule struct { + Field string `json:"field"` + Operator string `json:"operator"` + Value interface{} `json:"value,omitempty"` +} + +func (r Rule) Fields() []string { + return []string{r.Field} +} + +func (rg RuleGroup) Fields() []string { + var result []string + unique := map[string]struct{}{} + for _, r := range rg.Rules { + for _, f := range r.Fields() { + if _, added := unique[f]; !added { + result = append(result, f) + unique[f] = struct{}{} + } + } + } + return result +} + +func (rs *Rules) UnmarshalJSON(data []byte) error { + var rawRules []json.RawMessage + if err := json.Unmarshal(data, &rawRules); err != nil { + return err + } + rules := make(Rules, len(rawRules)) + for i, rawRule := range rawRules { + var r Rule + if err := json.Unmarshal(rawRule, &r); err == nil && r.Field != "" { + rules[i] = r + continue + } + var g RuleGroup + if err := json.Unmarshal(rawRule, &g); err == nil && g.Combinator != "" { + rules[i] = g + continue + } + return errors.New("Invalid json. Neither a Rule nor a RuleGroup: " + string(rawRule)) + } + *rs = rules + return nil +} + +var SmartPlaylistFields = []string{ + "title", + "album", + "artist", + "albumartist", + "albumartwork", + "tracknumber", + "discnumber", + "year", + "size", + "compilation", + "dateadded", + "datemodified", + "discsubtitle", + "comment", + "lyrics", + "sorttitle", + "sortalbum", + "sortartist", + "sortalbumartist", + "albumtype", + "albumcomment", + "catalognumber", + "filepath", + "filetype", + "duration", + "bitrate", + "bpm", + "channels", + "genre", + "loved", + "lastplayed", + "playcount", + "rating", +} diff --git a/model/smartplaylist_test.go b/model/smartplaylist_test.go new file mode 100644 index 000000000..2f2e73530 --- /dev/null +++ b/model/smartplaylist_test.go @@ -0,0 +1,101 @@ +package model_test + +import ( + "bytes" + "encoding/json" + + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("SmartPlaylist", func() { + var goObj model.SmartPlaylist + var jsonObj string + BeforeEach(func() { + goObj = model.SmartPlaylist{ + RuleGroup: model.RuleGroup{ + Combinator: "and", Rules: model.Rules{ + model.Rule{Field: "title", Operator: "contains", Value: "love"}, + model.Rule{Field: "year", Operator: "is in the range", Value: []int{1980, 1989}}, + model.Rule{Field: "loved", Operator: "is true"}, + model.Rule{Field: "lastPlayed", Operator: "in the last", Value: 30}, + model.RuleGroup{ + Combinator: "or", + Rules: model.Rules{ + model.Rule{Field: "artist", Operator: "is not", Value: "zé"}, + model.Rule{Field: "album", Operator: "is", Value: "4"}, + }, + }, + }}, + Order: "artist asc", + Limit: 100, + } + var b bytes.Buffer + err := json.Compact(&b, []byte(` +{ + "combinator":"and", + "rules":[ + { + "field":"title", + "operator":"contains", + "value":"love" + }, + { + "field":"year", + "operator":"is in the range", + "value":[ + 1980, + 1989 + ] + }, + { + "field":"loved", + "operator":"is true" + }, + { + "field":"lastPlayed", + "operator":"in the last", + "value":30 + }, + { + "combinator":"or", + "rules":[ + { + "field":"artist", + "operator":"is not", + "value":"zé" + }, + { + "field":"album", + "operator":"is", + "value":"4" + } + ] + } + ], + "order":"artist asc", + "limit":100 +}`)) + if err != nil { + panic(err) + } + jsonObj = b.String() + }) + It("finds all fields", func() { + Expect(goObj.Fields()).To(ConsistOf("title", "year", "loved", "lastPlayed", "artist", "album")) + }) + It("marshals to JSON", func() { + j, err := json.Marshal(goObj) + Expect(err).ToNot(HaveOccurred()) + Expect(string(j)).To(Equal(jsonObj)) + }) + It("is reversible to/from JSON", func() { + var newObj model.SmartPlaylist + err := json.Unmarshal([]byte(jsonObj), &newObj) + Expect(err).ToNot(HaveOccurred()) + j, err := json.Marshal(newObj) + Expect(err).ToNot(HaveOccurred()) + Expect(string(j)).To(Equal(jsonObj)) + }) +}) diff --git a/persistence/sql_smartplaylist.go b/persistence/sql_smartplaylist.go new file mode 100644 index 000000000..a788c414a --- /dev/null +++ b/persistence/sql_smartplaylist.go @@ -0,0 +1,273 @@ +package persistence + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + . "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" +) + +//{ +// "combinator": "and", +// "rules": [ +// {"field": "loved", "operator": "is true"}, +// {"field": "lastPlayed", "operator": "in the last", "value": "90"} +// ], +// "order": "artist asc", +// "limit": 100 +//} +type SmartPlaylist model.SmartPlaylist + +func (sp SmartPlaylist) AddFilters(sql SelectBuilder) SelectBuilder { + return sql.Where(RuleGroup(sp.RuleGroup)).OrderBy(sp.Order).Limit(uint64(sp.Limit)) +} + +type fieldDef struct { + dbField string + ruleType reflect.Type +} + +var fieldMap = map[string]*fieldDef{ + "title": {"media_file.title", stringRuleType}, + "album": {"media_file.album", stringRuleType}, + "artist": {"media_file.artist", stringRuleType}, + "albumartist": {"media_file.album_artist", stringRuleType}, + "albumartwork": {"media_file.has_cover_art", stringRuleType}, + "tracknumber": {"media_file.track_number", numberRuleType}, + "discnumber": {"media_file.disc_number", numberRuleType}, + "year": {"media_file.year", numberRuleType}, + "size": {"media_file.size", numberRuleType}, + "compilation": {"media_file.compilation", boolRuleType}, + "dateadded": {"media_file.created_at", dateRuleType}, + "datemodified": {"media_file.updated_at", dateRuleType}, + "discsubtitle": {"media_file.disc_subtitle", stringRuleType}, + "comment": {"media_file.comment", stringRuleType}, + "lyrics": {"media_file.lyrics", stringRuleType}, + "sorttitle": {"media_file.sort_title", stringRuleType}, + "sortalbum": {"media_file.sort_album_name", stringRuleType}, + "sortartist": {"media_file.sort_artist_name", stringRuleType}, + "sortalbumartist": {"media_file.sort_album_artist_name", stringRuleType}, + "albumtype": {"media_file.mbz_album_type", stringRuleType}, + "albumcomment": {"media_file.mbz_album_comment", stringRuleType}, + "catalognumber": {"media_file.catalog_num", stringRuleType}, + "filepath": {"media_file.path", stringRuleType}, + "filetype": {"media_file.suffix", stringRuleType}, + "duration": {"media_file.duration", numberRuleType}, + "bitrate": {"media_file.bit_rate", numberRuleType}, + "bpm": {"media_file.bpm", numberRuleType}, + "channels": {"media_file.channels", numberRuleType}, + "genre": {"genre.name", stringRuleType}, + "loved": {"annotation.starred", boolRuleType}, + "lastplayed": {"annotation.play_date", dateRuleType}, + "playcount": {"annotation.play_count", numberRuleType}, + "rating": {"annotation.rating", numberRuleType}, +} + +var stringRuleType = reflect.TypeOf(stringRule{}) + +type stringRule model.Rule + +func (r stringRule) ToSql() (sql string, args []interface{}, err error) { + var sq Sqlizer + switch r.Operator { + case "is": + sq = Eq{r.Field: r.Value} + case "is not": + sq = NotEq{r.Field: r.Value} + case "contains": + sq = ILike{r.Field: fmt.Sprintf("%%%s%%", r.Value)} + case "does not contains": + sq = NotILike{r.Field: fmt.Sprintf("%%%s%%", r.Value)} + case "begins with": + sq = ILike{r.Field: fmt.Sprintf("%s%%", r.Value)} + case "ends with": + sq = ILike{r.Field: fmt.Sprintf("%%%s", r.Value)} + default: + return "", nil, errors.New("operator not supported: " + r.Operator) + } + return sq.ToSql() +} + +var numberRuleType = reflect.TypeOf(numberRule{}) + +type numberRule model.Rule + +func (r numberRule) ToSql() (sql string, args []interface{}, err error) { + var sq Sqlizer + switch r.Operator { + case "is": + sq = Eq{r.Field: r.Value} + case "is not": + sq = NotEq{r.Field: r.Value} + case "is greater than": + sq = Gt{r.Field: r.Value} + case "is less than": + sq = Lt{r.Field: r.Value} + case "is in the range": + s := reflect.ValueOf(r.Value) + if s.Kind() != reflect.Slice || s.Len() != 2 { + return "", nil, fmt.Errorf("invalid range for 'in' operator: %s", r.Value) + } + sq = And{ + GtOrEq{r.Field: s.Index(0).Interface()}, + LtOrEq{r.Field: s.Index(1).Interface()}, + } + default: + return "", nil, errors.New("operator not supported: " + r.Operator) + } + return sq.ToSql() +} + +var dateRuleType = reflect.TypeOf(dateRule{}) + +type dateRule model.Rule + +func (r dateRule) ToSql() (string, []interface{}, error) { + var dates []time.Time + var err error + var sq Sqlizer + switch r.Operator { + case "is": + if dates, err = r.parseDates(); err != nil { + return "", nil, err + } + sq = Eq{r.Field: dates} + case "is not": + if dates, err = r.parseDates(); err != nil { + return "", nil, err + } + sq = NotEq{r.Field: dates} + case "is before": + if dates, err = r.parseDates(); err != nil { + return "", nil, err + } + sq = Lt{r.Field: dates[0]} + case "is after": + if dates, err = r.parseDates(); err != nil { + return "", nil, err + } + sq = Gt{r.Field: dates[0]} + case "is in the range": + if dates, err = r.parseDates(); err != nil { + return "", nil, err + } + if len(dates) != 2 { + return "", nil, fmt.Errorf("not a valid date range: %s", r.Value) + } + sq = And{Gt{r.Field: dates[0]}, Lt{r.Field: dates[1]}} + case "in the last": + sq, err = r.inTheLast(false) + if err != nil { + return "", nil, err + } + case "not in the last": + sq, err = r.inTheLast(true) + if err != nil { + return "", nil, err + } + default: + return "", nil, errors.New("operator not supported: " + r.Operator) + } + return sq.ToSql() +} + +func (r dateRule) inTheLast(invert bool) (Sqlizer, error) { + v, err := strconv.ParseInt(r.Value.(string), 10, 64) + if err != nil { + return nil, err + } + period := time.Now().Add(time.Duration(-24*v) * time.Hour) + if invert { + return Lt{r.Field: period}, nil + } + return Gt{r.Field: period}, nil +} + +func (r dateRule) parseDates() ([]time.Time, error) { + var input []string + switch v := r.Value.(type) { + case string: + input = append(input, v) + case []string: + input = append(input, v...) + } + var dates []time.Time + for _, s := range input { + d, err := time.Parse("2006-01-02", s) + if err != nil { + return nil, errors.New("invalid date: " + s) + } + dates = append(dates, d) + } + return dates, nil +} + +var boolRuleType = reflect.TypeOf(boolRule{}) + +type boolRule model.Rule + +func (r boolRule) ToSql() (sql string, args []interface{}, err error) { + var sq Sqlizer + switch r.Operator { + case "is true": + sq = Eq{r.Field: true} + case "is false": + sq = Eq{r.Field: false} + default: + return "", nil, errors.New("operator not supported: " + r.Operator) + } + return sq.ToSql() +} + +type RuleGroup model.RuleGroup + +func (rg RuleGroup) ToSql() (sql string, args []interface{}, err error) { + var sq []Sqlizer + for _, r := range rg.Rules { + switch rr := r.(type) { + case model.Rule: + sq = append(sq, rg.ruleToSqlizer(rr)) + case model.RuleGroup: + sq = append(sq, RuleGroup(rr)) + } + } + var group Sqlizer + if strings.ToLower(rg.Combinator) == "and" { + group = And(sq) + } else { + group = Or(sq) + } + return group.ToSql() +} + +type errorSqlizer string + +func (e errorSqlizer) ToSql() (sql string, args []interface{}, err error) { + return "", nil, errors.New(string(e)) +} + +func (rg RuleGroup) ruleToSqlizer(r model.Rule) Sqlizer { + ruleDef := fieldMap[strings.ToLower(r.Field)] + if ruleDef == nil { + return errorSqlizer("invalid smart playlist field " + r.Field) + } + r.Field = ruleDef.dbField + r.Operator = strings.ToLower(r.Operator) + switch ruleDef.ruleType { + case stringRuleType: + return stringRule(r) + case numberRuleType: + return numberRule(r) + case boolRuleType: + return boolRule(r) + case dateRuleType: + return dateRule(r) + default: + return errorSqlizer("invalid smart playlist rule type" + ruleDef.ruleType.String()) + } +} diff --git a/persistence/sql_smartplaylist_test.go b/persistence/sql_smartplaylist_test.go new file mode 100644 index 000000000..4454655d7 --- /dev/null +++ b/persistence/sql_smartplaylist_test.go @@ -0,0 +1,102 @@ +package persistence + +import ( + "time" + + "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/model" + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" +) + +var _ = Describe("SmartPlaylist", func() { + var pls SmartPlaylist + Describe("AddFilters", func() { + BeforeEach(func() { + sp := model.SmartPlaylist{ + RuleGroup: model.RuleGroup{ + Combinator: "and", Rules: model.Rules{ + model.Rule{Field: "title", Operator: "contains", Value: "love"}, + model.Rule{Field: "year", Operator: "is in the range", Value: []int{1980, 1989}}, + model.Rule{Field: "loved", Operator: "is true"}, + model.Rule{Field: "lastPlayed", Operator: "in the last", Value: "30"}, + model.RuleGroup{ + Combinator: "or", + Rules: model.Rules{ + model.Rule{Field: "artist", Operator: "is not", Value: "zé"}, + model.Rule{Field: "album", Operator: "is", Value: "4"}, + }, + }, + }}, + Order: "artist asc", + Limit: 100, + } + pls = SmartPlaylist(sp) + }) + + It("returns a proper SQL query", func() { + sel := pls.AddFilters(squirrel.Select("media_file").Columns("*")) + sql, args, err := sel.ToSql() + Expect(err).ToNot(HaveOccurred()) + Expect(sql).To(Equal("SELECT media_file, * WHERE (media_file.title ILIKE ? AND (media_file.year >= ? AND media_file.year <= ?) AND annotation.starred = ? AND annotation.play_date > ? AND (media_file.artist <> ? OR media_file.album = ?)) ORDER BY artist asc LIMIT 100")) + lastMonth := time.Now().Add(-30 * 24 * time.Hour) + Expect(args).To(ConsistOf("%love%", 1980, 1989, true, BeTemporally("~", lastMonth, time.Second), "zé", "4")) + }) + }) + + Describe("fieldMap", func() { + It("includes all possible fields", func() { + for _, field := range model.SmartPlaylistFields { + Expect(fieldMap).To(HaveKey(field)) + } + }) + It("does not have extra fields", func() { + for field := range fieldMap { + Expect(model.SmartPlaylistFields).To(ContainElement(field)) + } + }) + }) + + Describe("stringRule", func() { + DescribeTable("stringRule", + func(operator, expectedSql, expectedValue string) { + r := stringRule{Field: "title", Operator: operator, Value: "value"} + sql, args, err := r.ToSql() + Expect(err).ToNot(HaveOccurred()) + Expect(sql).To(Equal(expectedSql)) + Expect(args).To(ConsistOf(expectedValue)) + }, + Entry("is", "is", "title = ?", "value"), + Entry("is not", "is not", "title <> ?", "value"), + Entry("contains", "contains", "title ILIKE ?", "%value%"), + Entry("does not contains", "does not contains", "title NOT ILIKE ?", "%value%"), + Entry("begins with", "begins with", "title ILIKE ?", "value%"), + Entry("ends with", "ends with", "title ILIKE ?", "%value"), + ) + }) + + Describe("numberRule", func() { + DescribeTable("operators", + func(operator, expectedSql string, expectedValue ...interface{}) { + r := numberRule{Field: "year", Operator: operator, Value: 1985} + sql, args, err := r.ToSql() + Expect(err).ToNot(HaveOccurred()) + Expect(sql).To(Equal(expectedSql)) + Expect(args).To(ConsistOf(expectedValue...)) + }, + Entry("is", "is", "year = ?", 1985), + Entry("is not", "is not", "year <> ?", 1985), + Entry("is greater than", "is greater than", "year > ?", 1985), + Entry("is less than", "is less than", "year < ?", 1985), + ) + + It("implements the 'is in the range' operator", func() { + r := numberRule{Field: "year", Operator: "is in the range", Value: []int{1981, 1990}} + sql, args, err := r.ToSql() + Expect(err).ToNot(HaveOccurred()) + Expect(sql).To(Equal("(year >= ? AND year <= ?)")) + Expect(args).To(ConsistOf(1981, 1990)) + }) + }) +})