navidrome/db/migrations/20241026183640_support_new_scanner.go

package migrations

import (
	"context"
	"database/sql"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"testing/fstest"
	"unicode/utf8"

	"github.com/navidrome/navidrome/log"
	"github.com/navidrome/navidrome/model"
	"github.com/navidrome/navidrome/utils/chain"
	"github.com/pressly/goose/v3"
)

func init() {
	goose.AddMigrationContext(upSupportNewScanner, downSupportNewScanner)
}

func upSupportNewScanner(ctx context.Context, tx *sql.Tx) error {
	execute := createExecuteFunc(ctx, tx)
	addColumn := createAddColumnFunc(ctx, tx)

	return chain.RunSequentially(
		upSupportNewScanner_CreateTableFolder(ctx, execute),
		upSupportNewScanner_PopulateTableFolder(ctx, tx),
		upSupportNewScanner_UpdateTableMediaFile(ctx, execute, addColumn),
		upSupportNewScanner_UpdateTableAlbum(ctx, execute),
		upSupportNewScanner_UpdateTableArtist(ctx, execute, addColumn),
		execute(`
alter table library
	add column last_scan_started_at datetime default '0000-00-00 00:00:00' not null;
alter table library
	add column full_scan_in_progress boolean default false not null;

create table if not exists media_file_artists(
    	media_file_id varchar not null
				references media_file (id)
	    		 	on delete cascade,
    	artist_id varchar not null
				references artist (id)
	    		 	on delete cascade,
    	role varchar default '' not null,
    	sub_role varchar default '' not null,
    	constraint artist_tracks
    	    			unique (artist_id, media_file_id, role, sub_role)
);
create index if not exists media_file_artists_media_file_id
    on media_file_artists (media_file_id);
create index if not exists media_file_artists_role
	on media_file_artists (role);

create table if not exists album_artists(
    	album_id varchar not null
				references album (id)
	    		 	on delete cascade,
    	artist_id varchar not null
				references artist (id)
	    		 	on delete cascade,
    	role varchar default '' not null,
    	sub_role varchar default '' not null,
    	constraint album_artists
    	    			unique (album_id, artist_id, role, sub_role)
);
create index if not exists album_artists_album_id
    on album_artists (album_id);
create index if not exists album_artists_role
	on album_artists (role);

create table if not exists tag(
  	id varchar not null primary key,
  	tag_name varchar default '' not null,
  	tag_value varchar default '' not null,
  	album_count integer default 0 not null,
  	media_file_count integer default 0 not null,
  	constraint tags_name_value
		unique (tag_name, tag_value)
);

-- Genres are now stored in the tag table
drop table if exists media_file_genres;
drop table if exists album_genres;
drop table if exists artist_genres;
drop table if exists genre;

-- Drop full_text indexes, as they are not being used by SQLite
drop index if exists media_file_full_text;
drop index if exists album_full_text;
drop index if exists artist_full_text;

-- Add PID config to properties
insert into property (id, value) values ('PIDTrack', 'track_legacy') on conflict do nothing;
insert into property (id, value) values ('PIDAlbum', 'album_legacy') on conflict do nothing;
`),
		func() error {
			notice(tx, "A full scan will be triggered to populate the new tables. This may take a while.")
			return forceFullRescan(tx)
		},
	)
}

func upSupportNewScanner_CreateTableFolder(_ context.Context, execute execStmtFunc) execFunc {
	return execute(`
create table if not exists folder(
	id varchar not null
		primary key,
	library_id integer not null
	    		references library (id)
	    		 	on delete cascade,
	path varchar default '' not null,
	name varchar default '' not null,
	missing boolean default false not null,
	parent_id varchar default '' not null,
	num_audio_files integer default 0 not null,
	num_playlists integer default 0 not null,
	image_files jsonb default '[]' not null,
	images_updated_at datetime default '0000-00-00 00:00:00' not null,
	updated_at datetime default (datetime(current_timestamp, 'localtime')) not null,
	created_at datetime default (datetime(current_timestamp, 'localtime')) not null
);
create index folder_parent_id on folder(parent_id);
`)
}

// Use paths from `media_file` table to populate `folder` table. The `folder` table must contain all paths, including
// the ones that do not contain any media_file. We can get all paths from the media_file table to populate a
// fstest.MapFS{}, and then walk the filesystem to insert all folders into the DB, including empty parent ones.
func upSupportNewScanner_PopulateTableFolder(ctx context.Context, tx *sql.Tx) execFunc {
	return func() error {
		// First, get all folder paths from media_file table
		rows, err := tx.QueryContext(ctx, fmt.Sprintf(`
select distinct rtrim(media_file.path, replace(media_file.path, '%s', '')), library_id, library.path
from media_file
join library on media_file.library_id = library.id`, string(os.PathSeparator)))
		if err != nil {
			return err
		}
		defer rows.Close()

		// Then create an in-memory filesystem with all paths
		var path string
		var lib model.Library
		fsys := fstest.MapFS{}

		for rows.Next() {
			err = rows.Scan(&path, &lib.ID, &lib.Path)
			if err != nil {
				return err
			}

			path = strings.TrimPrefix(path, filepath.Clean(lib.Path))
			path = strings.TrimPrefix(path, string(os.PathSeparator))
			path = filepath.Clean(path)
			fsys[path] = &fstest.MapFile{Mode: fs.ModeDir}
		}
		if err = rows.Err(); err != nil {
			return fmt.Errorf("error loading folders from media_file table: %w", err)
		}
		if len(fsys) == 0 {
			return nil
		}

		stmt, err := tx.PrepareContext(ctx,
			"insert into folder (id, library_id, path, name, parent_id, updated_at) values (?, ?, ?, ?, ?, '0000-00-00 00:00:00')",
		)
		if err != nil {
			return err
		}

		// Finally, walk the in-mem filesystem and insert all folders into the DB.
		err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
			if err != nil {
				// Don't abort the walk, just log the error
				log.Error("error walking folder to DB", "path", path, err)
				return nil
			}
			// Skip entries that are not directories
			if !d.IsDir() {
				return nil
			}

			// Create a folder in the DB
			f := model.NewFolder(lib, path)
			_, err = stmt.ExecContext(ctx, f.ID, lib.ID, f.Path, f.Name, f.ParentID)
			if err != nil {
				log.Error("error writing folder to DB", "path", path, err)
			}
			return err
		})
		if err != nil {
			return fmt.Errorf("error populating folder table: %w", err)
		}

		// Count the number of characters in the library path
		libPath := filepath.Clean(lib.Path)
		libPathLen := utf8.RuneCountInString(libPath)

		// In one go, update all paths in the media_file table, removing the library path prefix
		// and replacing any backslashes with slashes (the path separator used by the io/fs package)
		_, err = tx.ExecContext(ctx, fmt.Sprintf(`
update media_file set path = replace(substr(path, %d), '\', '/');`, libPathLen+2))
		if err != nil {
			return fmt.Errorf("error updating media_file path: %w", err)
		}

		return nil
	}
}

func upSupportNewScanner_UpdateTableMediaFile(_ context.Context, execute execStmtFunc, addColumn addColumnFunc) execFunc {
	return func() error {
		return chain.RunSequentially(
			execute(`	
alter table media_file 
    add column folder_id varchar default '' not null;
alter table media_file 
    add column pid varchar default '' not null;
alter table media_file
	add column missing boolean default false not null;
alter table media_file
	add column mbz_release_group_id varchar default '' not null;
alter table media_file
	add column tags jsonb default '{}' not null;
alter table media_file
	add column participants jsonb default '{}' not null;
alter table media_file 
    add column bit_depth integer default 0 not null;
alter table media_file
	add column explicit_status varchar default '' not null;
`),
			addColumn("media_file", "birth_time", "datetime", "current_timestamp", "created_at"),
			execute(`	
update media_file 
	set pid = id where pid = '';
create index if not exists media_file_birth_time
	on media_file (birth_time);
create index if not exists media_file_folder_id
 	on media_file (folder_id);
create index if not exists media_file_pid
	on media_file (pid);
create index if not exists media_file_missing
	on media_file (missing);
`),
		)
	}
}

func upSupportNewScanner_UpdateTableAlbum(_ context.Context, execute execStmtFunc) execFunc {
	return execute(`
drop index if exists album_all_artist_ids;
alter table album
	drop column all_artist_ids;
drop index if exists album_artist;
drop index if exists album_artist_album;
alter table album
	drop column artist;
drop index if exists album_artist_id;
alter table album
	drop column artist_id;
alter table album
	add column imported_at datetime default '0000-00-00 00:00:00' not null;
alter table album
	add column missing boolean default false not null;
alter table album
	add column mbz_release_group_id varchar default '' not null;
alter table album
	add column tags jsonb default '{}' not null;
alter table album
	add column participants jsonb default '{}' not null;
alter table album
	drop column paths;
alter table album
	drop column image_files;
alter table album
	add column folder_ids jsonb default '[]' not null;
alter table album
	add column explicit_status varchar default '' not null; 
create index if not exists album_imported_at
	on album (imported_at);
create index if not exists album_mbz_release_group_id
	on album (mbz_release_group_id);
`)
}

func upSupportNewScanner_UpdateTableArtist(_ context.Context, execute execStmtFunc, addColumn addColumnFunc) execFunc {
	return func() error {
		return chain.RunSequentially(
			execute(`
alter table artist
	drop column album_count;
alter table artist
	drop column song_count;
drop index if exists artist_size;
alter table artist
	drop column size;
alter table artist
	add column missing boolean default false not null;
alter table artist
	add column stats jsonb default '{"albumartist":{}}' not null;
alter table artist
	drop column similar_artists;
alter table artist
	add column similar_artists jsonb default '[]' not null;
`),
			addColumn("artist", "updated_at", "datetime", "current_time", "(select min(album.updated_at) from album where album_artist_id = artist.id)"),
			addColumn("artist", "created_at", "datetime", "current_time", "(select min(album.created_at) from album where album_artist_id = artist.id)"),
			execute(`create index if not exists artist_updated_at on artist (updated_at);`),
			execute(`update artist set external_info_updated_at = '0000-00-00 00:00:00';`),
		)
	}
}

func downSupportNewScanner(context.Context, *sql.Tx) error {
	return nil
}