refactor: introduce GC, to delete old data

db/migrations/20200131183653_changes_for_new_persistence_layer.go

@@ -0,0 +1,63 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20200131183653, Down20200131183653)
+}
+
+func Up20200131183653(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create table search_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	item_type varchar(255) default '' not null,
+	full_text varchar(255) default '' not null
+);
+
+insert into search_dg_tmp(id, item_type, full_text) select id, "table", full_text from search;
+
+drop table search;
+
+alter table search_dg_tmp rename to search;
+
+create index search_full_text
+	on search (full_text);
+create index search_table
+	on search (item_type);
+
+update annotation set item_type = 'media_file' where item_type = 'mediaFile';
+`)
+	return err
+}
+
+func Down20200131183653(tx *sql.Tx) error {
+	tx.Exec(`
+create table search_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	"table" varchar(255) default '' not null,
+	full_text varchar(255) default '' not null
+);
+
+insert into search_dg_tmp(id, "table", full_text) select id, item_type, full_text from search;
+
+drop table search;
+
+alter table search_dg_tmp rename to search;
+
+create index search_full_text
+	on search (full_text);
+create index search_table
+	on search ("table");
+
+update annotation set item_type = 'mediaFile' where item_type = 'media_file';
+`)
+	return nil
+}

model/datastore.go

@@ -33,4 +33,5 @@ type DataStore interface {
 	Resource(ctx context.Context, model interface{}) ResourceRepository
 
 	WithTx(func(tx DataStore) error) error
+	GC(ctx context.Context) error
 }

persistence/mock_persistence.go

@@ -73,6 +73,10 @@ func (db *MockDataStore) Resource(ctx context.Context, m interface{}) model.Reso
 	return struct{ model.ResourceRepository }{}
 }
 
+func (db *MockDataStore) GC(ctx context.Context) error {
+	return nil
+}
+
 type mockedUserRepo struct {
 	model.UserRepository
 }

persistence/persistence.go

@@ -113,6 +113,26 @@ func (db *NewSQLStore) WithTx(block func(tx model.DataStore) error) error {
 	return nil
 }
 
+func (db *NewSQLStore) GC(ctx context.Context) error {
+	err := db.Album(ctx).PurgeEmpty()
+	if err != nil {
+		return err
+	}
+	err = db.Artist(ctx).PurgeEmpty()
+	if err != nil {
+		return err
+	}
+	err = db.MediaFile(ctx).(*mediaFileRepository).cleanSearchIndex()
+	if err != nil {
+		return err
+	}
+	err = db.Album(ctx).(*albumRepository).cleanSearchIndex()
+	if err != nil {
+		return err
+	}
+	return db.Artist(ctx).(*artistRepository).cleanSearchIndex()
+}
+
 func (db *NewSQLStore) getOrmer() orm.Ormer {
 	if db.orm == nil {
 		return orm.NewOrm()

persistence/search.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	. "github.com/Masterminds/squirrel"
+	"github.com/deluan/navidrome/log"
 	"github.com/kennygrant/sanitize"
 )
 
@@ -14,6 +15,7 @@ func (r sqlRepository) index(id string, text string) error {
 
 	values := map[string]interface{}{
 		"id":        id,
+		"item_type": r.tableName,
 		"full_text": sanitizedText,
 	}
 	update := Update(searchTable).Where(Eq{"id": id}).SetMap(values)
@@ -54,3 +56,15 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
 	_, err = r.ormer.Raw(sql, args...).QueryRows(results)
 	return err
 }
+
+func (r sqlRepository) cleanSearchIndex() error {
+	del := Delete(searchTable).Where(Eq{"item_type": r.tableName}).Where("id not in (select id from " + r.tableName + ")")
+	c, err := r.executeSQL(del)
+	if err != nil {
+		return err
+	}
+	if c > 0 {
+		log.Debug(r.ctx, "Clean-up search index", "table", r.tableName, "totalDeleted", c)
+	}
+	return nil
+}

scanner/tag_scanner.go

@@ -104,17 +104,11 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 		return err
 	}
 
-	err = s.ds.Album(ctx).PurgeEmpty()
+	if len(changed)+len(deleted) == 0 {
-	if err != nil {
+		return nil
-		return err
 	}
 
-	err = s.ds.Artist(ctx).PurgeEmpty()
+	return s.ds.GC(log.NewContext(nil))
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
 func (s *TagScanner) refreshAlbums(ctx context.Context, updatedAlbums map[string]bool) error {