From de1fea64bc6f01f36b034eb9fe87d2118a21f1bb Mon Sep 17 00:00:00 2001
From: Deluan <deluan@deluan.com>
Date: Fri, 31 Jan 2020 20:17:36 -0500
Subject: [PATCH] refactor: introduce GC, to delete old data

---
 ...83653_changes_for_new_persistence_layer.go | 63 +++++++++++++++++++
 model/datastore.go                            |  1 +
 persistence/mock_persistence.go               |  4 ++
 persistence/persistence.go                    | 20 ++++++
 persistence/search.go                         | 14 +++++
 scanner/tag_scanner.go                        | 12 +---
 6 files changed, 105 insertions(+), 9 deletions(-)
 create mode 100644 db/migrations/20200131183653_changes_for_new_persistence_layer.go

diff --git a/db/migrations/20200131183653_changes_for_new_persistence_layer.go b/db/migrations/20200131183653_changes_for_new_persistence_layer.go
new file mode 100644
index 000000000..5f1462fe6
--- /dev/null
+++ b/db/migrations/20200131183653_changes_for_new_persistence_layer.go
@@ -0,0 +1,63 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20200131183653, Down20200131183653)
+}
+
+func Up20200131183653(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create table search_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	item_type varchar(255) default '' not null,
+	full_text varchar(255) default '' not null
+);
+
+insert into search_dg_tmp(id, item_type, full_text) select id, "table", full_text from search;
+
+drop table search;
+
+alter table search_dg_tmp rename to search;
+
+create index search_full_text
+	on search (full_text);
+create index search_table
+	on search (item_type);
+
+update annotation set item_type = 'media_file' where item_type = 'mediaFile';
+`)
+	return err
+}
+
+func Down20200131183653(tx *sql.Tx) error {
+	tx.Exec(`
+create table search_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	"table" varchar(255) default '' not null,
+	full_text varchar(255) default '' not null
+);
+
+insert into search_dg_tmp(id, "table", full_text) select id, item_type, full_text from search;
+
+drop table search;
+
+alter table search_dg_tmp rename to search;
+
+create index search_full_text
+	on search (full_text);
+create index search_table
+	on search ("table");
+
+update annotation set item_type = 'mediaFile' where item_type = 'media_file';
+`)
+	return nil
+}
diff --git a/model/datastore.go b/model/datastore.go
index 37cd352b9..b38fe1757 100644
--- a/model/datastore.go
+++ b/model/datastore.go
@@ -33,4 +33,5 @@ type DataStore interface {
 	Resource(ctx context.Context, model interface{}) ResourceRepository
 
 	WithTx(func(tx DataStore) error) error
+	GC(ctx context.Context) error
 }
diff --git a/persistence/mock_persistence.go b/persistence/mock_persistence.go
index ac021a526..f2069dfd1 100644
--- a/persistence/mock_persistence.go
+++ b/persistence/mock_persistence.go
@@ -73,6 +73,10 @@ func (db *MockDataStore) Resource(ctx context.Context, m interface{}) model.Reso
 	return struct{ model.ResourceRepository }{}
 }
 
+func (db *MockDataStore) GC(ctx context.Context) error {
+	return nil
+}
+
 type mockedUserRepo struct {
 	model.UserRepository
 }
diff --git a/persistence/persistence.go b/persistence/persistence.go
index 91c571d56..2f24a2f41 100644
--- a/persistence/persistence.go
+++ b/persistence/persistence.go
@@ -113,6 +113,26 @@ func (db *NewSQLStore) WithTx(block func(tx model.DataStore) error) error {
 	return nil
 }
 
+func (db *NewSQLStore) GC(ctx context.Context) error {
+	err := db.Album(ctx).PurgeEmpty()
+	if err != nil {
+		return err
+	}
+	err = db.Artist(ctx).PurgeEmpty()
+	if err != nil {
+		return err
+	}
+	err = db.MediaFile(ctx).(*mediaFileRepository).cleanSearchIndex()
+	if err != nil {
+		return err
+	}
+	err = db.Album(ctx).(*albumRepository).cleanSearchIndex()
+	if err != nil {
+		return err
+	}
+	return db.Artist(ctx).(*artistRepository).cleanSearchIndex()
+}
+
 func (db *NewSQLStore) getOrmer() orm.Ormer {
 	if db.orm == nil {
 		return orm.NewOrm()
diff --git a/persistence/search.go b/persistence/search.go
index 404872738..1d88ed8fc 100644
--- a/persistence/search.go
+++ b/persistence/search.go
@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	. "github.com/Masterminds/squirrel"
+	"github.com/deluan/navidrome/log"
 	"github.com/kennygrant/sanitize"
 )
 
@@ -14,6 +15,7 @@ func (r sqlRepository) index(id string, text string) error {
 
 	values := map[string]interface{}{
 		"id":        id,
+		"item_type": r.tableName,
 		"full_text": sanitizedText,
 	}
 	update := Update(searchTable).Where(Eq{"id": id}).SetMap(values)
@@ -54,3 +56,15 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
 	_, err = r.ormer.Raw(sql, args...).QueryRows(results)
 	return err
 }
+
+func (r sqlRepository) cleanSearchIndex() error {
+	del := Delete(searchTable).Where(Eq{"item_type": r.tableName}).Where("id not in (select id from " + r.tableName + ")")
+	c, err := r.executeSQL(del)
+	if err != nil {
+		return err
+	}
+	if c > 0 {
+		log.Debug(r.ctx, "Clean-up search index", "table", r.tableName, "totalDeleted", c)
+	}
+	return nil
+}
diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go
index 60f6621f1..7308f9414 100644
--- a/scanner/tag_scanner.go
+++ b/scanner/tag_scanner.go
@@ -104,17 +104,11 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 		return err
 	}
 
-	err = s.ds.Album(ctx).PurgeEmpty()
-	if err != nil {
-		return err
+	if len(changed)+len(deleted) == 0 {
+		return nil
 	}
 
-	err = s.ds.Artist(ctx).PurgeEmpty()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return s.ds.GC(log.NewContext(nil))
 }
 
 func (s *TagScanner) refreshAlbums(ctx context.Context, updatedAlbums map[string]bool) error {