mirror of
https://github.com/navidrome/navidrome.git
synced 2025-04-03 20:47:35 +03:00
feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)
* fix(server): more race conditions when updating artist/album from external sources
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): pass configfile option to child process
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): resume interrupted fullScans
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): remove old scanner code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): rename old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): move old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: tests
Signed-off-by: Deluan <deluan@navidrome.org>
* chore(deps): update Go to 1.23.4
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(test):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log level
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add config for scanner watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: children playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: replace `interface{}` with `any`
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlists with genres
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow any tags in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist names in playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlist's sort by tags
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use generic JSONArray for OS arrays
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use https in test
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add releaseTypes to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add recordLabels to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): rename JSONArray to Array
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): do not pre-populate smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement a simplified version of ArtistID3.
See https://github.com/opensubsonic/open-subsonic-api/discussions/120
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add contributors to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add albumArtists to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayArtist and displayAlbumArtist
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayComposer to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add roles to ArtistID3
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): use " • " separator for displayComposer
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): respect `PreferSortTags` config option
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize purging non-unused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: don't run 'refresh artist stats' concurrently with other transactions
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add Scanner.ScanOnStartup config option, default true
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: better json parsing error msg when importing NSPs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update album's imported_time when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: handle interrupted scans and full scans after migrations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `analyze` when migration requires a full rescan
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `PRAGMA optimize` at the end of the scan
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update artist's updated_at when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: handle multiple artists and roles in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): dim missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album missing logic
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error encoding in gob
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: separate warnings from errors
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: mark albums as missing if they were contained in a deleted folder
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: add participant names to media_file and album tables
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: use participations in criteria, instead of m2m relationship
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: rename participations to participants
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: albumartist role case
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): run scanner as an external process by default
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): show albumArtist names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): dim out missing albums
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: flaky test
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(server): scrobble buffer mapping. fix #3583
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: more participations renaming
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: send release_group_mbid to listenbrainz
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement OpenSubsonic explicitStatus field (#3597)
* feat: implement OpenSubsonic explicitStatus field
* fix(subsonic): fix failing snapshot tests
* refactor: create helper for setting explicitStatus
* fix: store smaller values for explicit-status on database
* test: ToAlbum explicitStatus
* refactor: rename explicitStatus helper function
---------
Co-authored-by: Deluan Quintão <deluan@navidrome.org>
* fix: handle album and track tags in the DB based on the mappings.yaml file
Signed-off-by: Deluan <deluan@navidrome.org>
* save similar artists as JSONB
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: getAlbumList byGenre
Signed-off-by: Deluan <deluan@navidrome.org>
* detect changes in PID configuration
Signed-off-by: Deluan <deluan@navidrome.org>
* set default album PID to legacy_pid
Signed-off-by: Deluan <deluan@navidrome.org>
* fix tests
Signed-off-by: Deluan <deluan@navidrome.org>
* fix SIGSEGV
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't lose album stars/ratings when migrating
Signed-off-by: Deluan <deluan@navidrome.org>
* store full PID conf in properties
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: keep album annotations when changing PID.Album config
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reassign album annotations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: use (display) albumArtist and add links to each artist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: not showing albums by albumartist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error msgs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: hide PID from Native API
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album cover art resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim participant names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reduce watcher log spam
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: panic when initializing the watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: various artists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't store empty lyrics in the DB
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused methods
Signed-off-by: Deluan <deluan@navidrome.org>
* drop full_text indexes, as they are not being used by SQLite
Signed-off-by: Deluan <deluan@navidrome.org>
* keep album created_at when upgrading
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null pointer
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album artwork cache
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't expose missing files in Subsonic API
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: searchable interface
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: filter out missing items from subsonic search
* fix: filter out missing items from playlists
* fix: filter out missing items from shares
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): add filter by artist role
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): only return albumartists in getIndexes and getArtists endpoints
Signed-off-by: Deluan <deluan@navidrome.org>
* sort roles alphabetically
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist playcounts
Signed-off-by: Deluan <deluan@navidrome.org>
* change default Album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix albumartist link when it does not match any albumartists values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix `Ignoring filter not whitelisted` (role) message
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim any names/titles being imported
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused genre code
Signed-off-by: Deluan <deluan@navidrome.org>
* serialize calls to Last.fm's getArtist
Signed-off-by: Deluan <deluan@navidrome.org>
xxx
Signed-off-by: Deluan <deluan@navidrome.org>
* add counters to genres
Signed-off-by: Deluan <deluan@navidrome.org>
* nit: fix migration `notice` message
Signed-off-by: Deluan <deluan@navidrome.org>
* optimize similar artists query
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: last.fm.getInfo when mbid does not exist
Signed-off-by: Deluan <deluan@navidrome.org>
* ui only show missing items for admins
Signed-off-by: Deluan <deluan@navidrome.org>
* don't allow interaction with missing items
Signed-off-by: Deluan <deluan@navidrome.org>
* Add Missing Files view (WIP)
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: merged tag_counts into tag table
Signed-off-by: Deluan <deluan@navidrome.org>
* add option to completely disable automatic scanner
Signed-off-by: Deluan <deluan@navidrome.org>
* add delete missing files functionality
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: playlists not showing for regular users
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce updateLastAccess frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce update player frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* add timeout when updating player
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dead code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix duplicated roles in stats
Signed-off-by: Deluan <deluan@navidrome.org>
* add `; ` to artist splitters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix stats query
Signed-off-by: Deluan <deluan@navidrome.org>
* more logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* add record label filter
Signed-off-by: Deluan <deluan@navidrome.org>
* add release type filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix purgeUnused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* add grouping filter to albums
Signed-off-by: Deluan <deluan@navidrome.org>
* allow any album tags to be used in as filters in the API
Signed-off-by: Deluan <deluan@navidrome.org>
* remove empty tags from album info
Signed-off-by: Deluan <deluan@navidrome.org>
* comments in the migration
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: Cannot read properties of undefined
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling (#3640)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove duplicated tag values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't ignore the taglib folder!
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: show track subtitle tag
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: show artists stats based on selected role
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: inspect
Signed-off-by: Deluan <deluan@navidrome.org>
* add media type to album info/filters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: change format of subtitle in the UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in Subsonic API and search
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in UI's player
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: split strings should be case-insensitive
Signed-off-by: Deluan <deluan@navidrome.org>
* disable ScanSchedule
Signed-off-by: Deluan <deluan@navidrome.org>
* increase default sessiontimeout
Signed-off-by: Deluan <deluan@navidrome.org>
* add sqlite command line tool to docker image
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: resources override
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* change migration to mark current artists as albumArtists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Allow filtering on multiple genres (#3679)
* feat(ui): Allow filtering on multiple genres
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
* add multi-genre filter in Album list
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
* add more multi-valued tag filters to Album and Song views
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): unselect missing files after removing
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): song filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix sharing tracks. fix #3687
Signed-off-by: Deluan <deluan@navidrome.org>
* use rowids when using search for sync (ex: Symfonium)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients for search
Signed-off-by: Deluan <deluan@navidrome.org>
* add libraryPath to Native API /songs endpoint
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add album version
Signed-off-by: Deluan <deluan@navidrome.org>
* made all tags lowercase as they are case-insensitive anyways.
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Show full paths, extended properties for album/song (#3691)
* feat(ui): Show full paths, extended properties for album/song
- uses library path + os separator + path
- show participants (album/song) and tags (song)
- make album/participant clickable in show info
* add source to path
* fix pathSeparator in UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix local artist artwork (#3695)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: parse vorbis performers
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: clean function into smaller functions
Signed-off-by: Deluan <deluan@navidrome.org>
* fix translations for en and pt
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow performers without instrument/subrole
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: metadata clean function again
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize split function
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: split function is now a method of TagConf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: humanize Artist total size
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version to album details
Signed-off-by: Deluan <deluan@navidrome.org>
* don't display album-level tags in SongInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* fix genre clicking in Album Page
Signed-off-by: Deluan <deluan@navidrome.org>
* don't use mbids in Last.fm api calls.
From 1337574018:
With MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo
{
artist: {
name: "Bee Gees",
mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810",
url: "https://www.last.fm/music/Bee+Gees",
}
```
Without MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo
{
artist: {
name: "Van Morrison",
mbid: "a41ac10f-0a56-4672-9161-b83f9b223559",
url: "https://www.last.fm/music/Van+Morrison",
}
```
Signed-off-by: Deluan <deluan@navidrome.org>
* better logging for when the artist folder is not found
Signed-off-by: Deluan <deluan@navidrome.org>
* fix various issues with artist image resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* hide "Additional Tags" header if there are none.
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify tag rendering
Signed-off-by: Deluan <deluan@navidrome.org>
* enhance logging for artist folder detection
Signed-off-by: Deluan <deluan@navidrome.org>
* make folderID consistent for relative and absolute folderPaths
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more folder paths scenarios
Signed-off-by: Deluan <deluan@navidrome.org>
* filter out other roles when SubsonicArtistParticipations = true
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Cannot read properties of undefined"
Signed-off-by: Deluan <deluan@navidrome.org>
* fix lyrics and comments being truncated (#3701)
* fix lyrics and comments being truncated
* specifically test for lyrics and comment length
* reorder assertions
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* fix(server): Expose library_path for playlist (#3705)
Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic)
* fix BFR on Windows (#3704)
* fix potential reflected cross-site scripting vulnerability
Signed-off-by: Deluan <deluan@navidrome.org>
* hack to make it work on Windows
* ignore windows executables
* try fixing the pipeline
Signed-off-by: Deluan <deluan@navidrome.org>
* allow MusicFolder in other drives
* move windows local drive logic to local storage implementation
---------
Signed-off-by: Deluan <deluan@navidrome.org>
* increase pagination sizes for missing files
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce level of "already scanning" watcher log message
Signed-off-by: Deluan <deluan@navidrome.org>
* only count folders with audio files in it
See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version and catalog number to search
Signed-off-by: Deluan <deluan@navidrome.org>
* add `organization` alias for `recordlabel`
Signed-off-by: Deluan <deluan@navidrome.org>
* remove mbid from Last.fm agent
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: support inspect in ui (#3726)
* inspect in ui
* address round 1
* add catalogNum to AlbumInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dependency on metadata_old (deprecated) package
Signed-off-by: Deluan <deluan@navidrome.org>
* add `RawTags` to model
Signed-off-by: Deluan <deluan@navidrome.org>
* support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698)
* parse standard roles, vorbis/m4a work for now
* fix djmixer
* working roles, use DJ-mix
* add performers to file
* map mbids
* add a few more tests
* add test
Signed-off-by: Deluan <deluan@navidrome.org>
* try to simplify the performers logic
Signed-off-by: Deluan <deluan@navidrome.org>
* stylistic changes
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* remove param mutation
Signed-off-by: Deluan <deluan@navidrome.org>
* run automated SQLite optimizations
Signed-off-by: Deluan <deluan@navidrome.org>
* fix playlists import/export on Windows
* fix import playlists
* fix export playlists
* better handling of Windows volumes
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more album ID reassignments
Signed-off-by: Deluan <deluan@navidrome.org>
* allow adding/overriding tags in the config file
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): Fix playlist track id, handle missing tracks better (#3734)
- Use `mediaFileId` instead of `id` for playlist tracks
- Only fetch if the file is not missing
- If extractor fails to get the file, also error (rather than panic)
* optimize DB after each scan.
Signed-off-by: Deluan <deluan@navidrome.org>
* remove sortable from AlbumSongs columns
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify query to get missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* mark Scanner.Extractor as deprecated
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Caio Cotts <caio@cotts.com.br>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão
GitHub
parent
46a963a02a
commit
c795bcfcf7
329 changed files with 16586 additions and 5852 deletions
|
|
@ -2,264 +2,243 @@ package scanner
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/utils/singleton"
|
||||
"golang.org/x/time/rate"
|
||||
"github.com/navidrome/navidrome/utils/chain"
|
||||
)
|
||||
|
||||
type Scanner interface {
|
||||
RescanAll(ctx context.Context, fullRescan bool) error
|
||||
Status(library string) (*StatusInfo, error)
|
||||
type scannerImpl struct {
|
||||
ds model.DataStore
|
||||
cw artwork.CacheWarmer
|
||||
pls core.Playlists
|
||||
metrics metrics.Metrics
|
||||
}
|
||||
|
||||
type StatusInfo struct {
|
||||
Library string
|
||||
Scanning bool
|
||||
LastScan time.Time
|
||||
Count uint32
|
||||
FolderCount uint32
|
||||
// scanState holds the state of an in-progress scan, to be passed to the various phases
|
||||
type scanState struct {
|
||||
progress chan<- *ProgressInfo
|
||||
fullScan bool
|
||||
changesDetected atomic.Bool
|
||||
}
|
||||
|
||||
var (
|
||||
ErrAlreadyScanning = errors.New("already scanning")
|
||||
ErrScanError = errors.New("scan error")
|
||||
)
|
||||
|
||||
type FolderScanner interface {
|
||||
// Scan process finds any changes after `lastModifiedSince` and returns the number of changes found
|
||||
Scan(ctx context.Context, lib model.Library, fullRescan bool, progress chan uint32) (int64, error)
|
||||
}
|
||||
|
||||
var isScanning sync.Mutex
|
||||
|
||||
type scanner struct {
|
||||
once sync.Once
|
||||
folders map[string]FolderScanner
|
||||
libs map[string]model.Library
|
||||
status map[string]*scanStatus
|
||||
lock *sync.RWMutex
|
||||
ds model.DataStore
|
||||
pls core.Playlists
|
||||
broker events.Broker
|
||||
cacheWarmer artwork.CacheWarmer
|
||||
metrics metrics.Metrics
|
||||
}
|
||||
|
||||
type scanStatus struct {
|
||||
active bool
|
||||
fileCount uint32
|
||||
folderCount uint32
|
||||
lastUpdate time.Time
|
||||
}
|
||||
|
||||
func GetInstance(ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer, broker events.Broker, metrics metrics.Metrics) Scanner {
|
||||
return singleton.GetInstance(func() *scanner {
|
||||
s := &scanner{
|
||||
ds: ds,
|
||||
pls: playlists,
|
||||
broker: broker,
|
||||
folders: map[string]FolderScanner{},
|
||||
libs: map[string]model.Library{},
|
||||
status: map[string]*scanStatus{},
|
||||
lock: &sync.RWMutex{},
|
||||
cacheWarmer: cacheWarmer,
|
||||
metrics: metrics,
|
||||
}
|
||||
s.loadFolders()
|
||||
return s
|
||||
})
|
||||
}
|
||||
|
||||
func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) error {
|
||||
folderScanner := s.folders[library]
|
||||
start := time.Now()
|
||||
|
||||
lib, ok := s.libs[library]
|
||||
if !ok {
|
||||
log.Error(ctx, "Folder not a valid library path", "folder", library)
|
||||
return fmt.Errorf("folder %s not a valid library path", library)
|
||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
if s.progress != nil {
|
||||
s.progress <- info
|
||||
}
|
||||
}
|
||||
|
||||
s.setStatusStart(library)
|
||||
defer s.setStatusEnd(library, start)
|
||||
func (s *scanState) sendWarning(msg string) {
|
||||
s.sendProgress(&ProgressInfo{Warning: msg})
|
||||
}
|
||||
|
||||
if fullRescan {
|
||||
log.Debug("Scanning folder (full scan)", "folder", library)
|
||||
} else {
|
||||
log.Debug("Scanning folder", "folder", library, "lastScan", lib.LastScanAt)
|
||||
}
|
||||
func (s *scanState) sendError(err error) {
|
||||
s.sendProgress(&ProgressInfo{Error: err.Error()})
|
||||
}
|
||||
|
||||
progress, cancel := s.startProgressTracker(library)
|
||||
defer cancel()
|
||||
|
||||
changeCount, err := folderScanner.Scan(ctx, lib, fullRescan, progress)
|
||||
func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
|
||||
state := scanState{progress: progress, fullScan: fullScan}
|
||||
libs, err := s.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
log.Error("Error scanning Library", "folder", library, err)
|
||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
if changeCount > 0 {
|
||||
log.Debug(ctx, "Detected changes in the music folder. Sending refresh event",
|
||||
"folder", library, "changeCount", changeCount)
|
||||
// Don't use real context, forcing a refresh in all open windows, including the one that triggered the scan
|
||||
s.broker.SendMessage(context.Background(), &events.RefreshResource{})
|
||||
}
|
||||
startTime := time.Now()
|
||||
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs))
|
||||
|
||||
s.updateLastModifiedSince(ctx, library, start)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *scanner) startProgressTracker(library string) (chan uint32, context.CancelFunc) {
|
||||
// Must be a new context (not the one passed to the scan method) to allow broadcasting the scan status to all clients
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
progress := make(chan uint32, 1000)
|
||||
limiter := rate.Sometimes{Interval: conf.Server.DevActivityPanelUpdateRate}
|
||||
go func() {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{Scanning: true, Count: 0, FolderCount: 0})
|
||||
defer func() {
|
||||
if status, ok := s.getStatus(library); ok {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{
|
||||
Scanning: false,
|
||||
Count: int64(status.fileCount),
|
||||
FolderCount: int64(status.folderCount),
|
||||
})
|
||||
}
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case count := <-progress:
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
totalFolders, totalFiles := s.incStatusCounter(library, count)
|
||||
limiter.Do(func() {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{
|
||||
Scanning: true,
|
||||
Count: int64(totalFiles),
|
||||
FolderCount: int64(totalFolders),
|
||||
})
|
||||
})
|
||||
// if there was a full scan in progress, force a full scan
|
||||
if !state.fullScan {
|
||||
for _, lib := range libs {
|
||||
if lib.FullScanInProgress {
|
||||
log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name)
|
||||
state.fullScan = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return progress, cancel
|
||||
}
|
||||
|
||||
func (s *scanner) getStatus(folder string) (scanStatus, bool) {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
status, ok := s.status[folder]
|
||||
return *status, ok
|
||||
}
|
||||
|
||||
func (s *scanner) incStatusCounter(folder string, numFiles uint32) (totalFolders uint32, totalFiles uint32) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.fileCount += numFiles
|
||||
status.folderCount++
|
||||
totalFolders = status.folderCount
|
||||
totalFiles = status.fileCount
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *scanner) setStatusStart(folder string) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.active = true
|
||||
status.fileCount = 0
|
||||
status.folderCount = 0
|
||||
}
|
||||
}
|
||||
err = chain.RunSequentially(
|
||||
// Phase 1: Scan all libraries and import new/updated files
|
||||
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)),
|
||||
|
||||
func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.active = false
|
||||
status.lastUpdate = lastUpdate
|
||||
}
|
||||
}
|
||||
// Phase 2: Process missing files, checking for moves
|
||||
runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)),
|
||||
|
||||
func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
s.once.Do(s.loadFolders)
|
||||
// Phases 3 and 4 can be run in parallel
|
||||
chain.RunParallel(
|
||||
// Phase 3: Refresh all new/changed albums and update artists
|
||||
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)),
|
||||
|
||||
if !isScanning.TryLock() {
|
||||
log.Debug(ctx, "Scanner already running, ignoring request for rescan.")
|
||||
return ErrAlreadyScanning
|
||||
}
|
||||
defer isScanning.Unlock()
|
||||
// Phase 4: Import/update playlists
|
||||
runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)),
|
||||
),
|
||||
|
||||
var hasError bool
|
||||
for folder := range s.folders {
|
||||
err := s.rescan(ctx, folder, fullRescan)
|
||||
hasError = hasError || err != nil
|
||||
}
|
||||
if hasError {
|
||||
log.Error(ctx, "Errors while scanning media. Please check the logs")
|
||||
// Final Steps (cannot be parallelized):
|
||||
|
||||
// Run GC if there were any changes (Remove dangling tracks, empty albums and artists, and orphan annotations)
|
||||
s.runGC(ctx, &state),
|
||||
|
||||
// Refresh artist and tags stats
|
||||
s.runRefreshStats(ctx, &state),
|
||||
|
||||
// Update last_scan_completed_at for all libraries
|
||||
s.runUpdateLibraries(ctx, libs),
|
||||
|
||||
// Optimize DB
|
||||
s.runOptimize(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err)
|
||||
state.sendError(err)
|
||||
s.metrics.WriteAfterScanMetrics(ctx, false)
|
||||
return ErrScanError
|
||||
return
|
||||
}
|
||||
s.metrics.WriteAfterScanMetrics(ctx, true)
|
||||
return nil
|
||||
|
||||
if state.changesDetected.Load() {
|
||||
state.sendProgress(&ProgressInfo{ChangesDetected: true})
|
||||
}
|
||||
|
||||
s.metrics.WriteAfterScanMetrics(ctx, err == nil)
|
||||
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
|
||||
}
|
||||
|
||||
func (s *scanner) Status(library string) (*StatusInfo, error) {
|
||||
s.once.Do(s.loadFolders)
|
||||
status, ok := s.getStatus(library)
|
||||
if !ok {
|
||||
return nil, errors.New("library not found")
|
||||
func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error {
|
||||
return func() error {
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
if state.changesDetected.Load() {
|
||||
start := time.Now()
|
||||
err := tx.GC(ctx)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error running GC", err)
|
||||
return fmt.Errorf("running GC: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: GC completed", "elapsed", time.Since(start))
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: No changes detected, skipping GC")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return &StatusInfo{
|
||||
Library: library,
|
||||
Scanning: status.active,
|
||||
LastScan: status.lastUpdate,
|
||||
Count: status.fileCount,
|
||||
FolderCount: status.folderCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *scanner) updateLastModifiedSince(ctx context.Context, folder string, t time.Time) {
|
||||
lib := s.libs[folder]
|
||||
id := lib.ID
|
||||
if err := s.ds.Library(ctx).UpdateLastScan(id, t); err != nil {
|
||||
log.Error("Error updating DB after scan", err)
|
||||
}
|
||||
lib.LastScanAt = t
|
||||
s.libs[folder] = lib
|
||||
}
|
||||
|
||||
func (s *scanner) loadFolders() {
|
||||
ctx := context.TODO()
|
||||
libs, _ := s.ds.Library(ctx).GetAll()
|
||||
for _, lib := range libs {
|
||||
log.Info("Configuring Media Folder", "name", lib.Name, "path", lib.Path)
|
||||
s.folders[lib.Path] = s.newScanner()
|
||||
s.libs[lib.Path] = lib
|
||||
s.status[lib.Path] = &scanStatus{
|
||||
active: false,
|
||||
fileCount: 0,
|
||||
folderCount: 0,
|
||||
lastUpdate: lib.LastScanAt,
|
||||
func (s *scannerImpl) runRefreshStats(ctx context.Context, state *scanState) func() error {
|
||||
return func() error {
|
||||
if !state.changesDetected.Load() {
|
||||
log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats")
|
||||
return nil
|
||||
}
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
start := time.Now()
|
||||
stats, err := tx.Artist(ctx).RefreshStats()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error refreshing artists stats", err)
|
||||
return fmt.Errorf("refreshing artists stats: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
|
||||
|
||||
start = time.Now()
|
||||
err = tx.Tag(ctx).UpdateCounts()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating tag counts", err)
|
||||
return fmt.Errorf("updating tag counts: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scanner) newScanner() FolderScanner {
|
||||
return NewTagScanner(s.ds, s.pls, s.cacheWarmer)
|
||||
func (s *scannerImpl) runOptimize(ctx context.Context) func() error {
|
||||
return func() error {
|
||||
start := time.Now()
|
||||
db.Optimize(ctx)
|
||||
log.Debug(ctx, "Scanner: Optimized DB", "elapsed", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scannerImpl) runUpdateLibraries(ctx context.Context, libs model.Libraries) func() error {
|
||||
return func() error {
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
for _, lib := range libs {
|
||||
err := tx.Library(ctx).ScanEnd(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating last scan completed", "lib", lib.Name, err)
|
||||
return fmt.Errorf("updating last scan completed: %w", err)
|
||||
}
|
||||
err = tx.Property(ctx).Put(consts.PIDTrackKey, conf.Server.PID.Track)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating track PID conf", err)
|
||||
return fmt.Errorf("updating track PID conf: %w", err)
|
||||
}
|
||||
err = tx.Property(ctx).Put(consts.PIDAlbumKey, conf.Server.PID.Album)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating album PID conf", err)
|
||||
return fmt.Errorf("updating album PID conf: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type phase[T any] interface {
|
||||
producer() ppl.Producer[T]
|
||||
stages() []ppl.Stage[T]
|
||||
finalize(error) error
|
||||
description() string
|
||||
}
|
||||
|
||||
func runPhase[T any](ctx context.Context, phaseNum int, phase phase[T]) func() error {
|
||||
return func() error {
|
||||
log.Debug(ctx, fmt.Sprintf("Scanner: Starting phase %d: %s", phaseNum, phase.description()))
|
||||
start := time.Now()
|
||||
|
||||
producer := phase.producer()
|
||||
stages := phase.stages()
|
||||
|
||||
// Prepend a counter stage to the phase's pipeline
|
||||
counter, countStageFn := countTasks[T]()
|
||||
stages = append([]ppl.Stage[T]{ppl.NewStage(countStageFn, ppl.Name("count tasks"))}, stages...)
|
||||
|
||||
var err error
|
||||
if log.IsGreaterOrEqualTo(log.LevelDebug) {
|
||||
var m *ppl.Metrics
|
||||
m, err = ppl.Measure(producer, stages...)
|
||||
log.Info(ctx, "Scanner: "+m.String(), err)
|
||||
} else {
|
||||
err = ppl.Do(producer, stages...)
|
||||
}
|
||||
|
||||
err = phase.finalize(err)
|
||||
|
||||
if err != nil {
|
||||
log.Error(ctx, fmt.Sprintf("Scanner: Error processing libraries in phase %d", phaseNum), "elapsed", time.Since(start), err)
|
||||
} else {
|
||||
log.Debug(ctx, fmt.Sprintf("Scanner: Finished phase %d", phaseNum), "elapsed", time.Since(start), "totalTasks", counter.Load())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func countTasks[T any]() (*atomic.Int64, func(T) (T, error)) {
|
||||
counter := atomic.Int64{}
|
||||
return &counter, func(in T) (T, error) {
|
||||
counter.Add(1)
|
||||
return in, nil
|
||||
}
|
||||
}
|
||||
|
||||
var _ scanner = (*scannerImpl)(nil)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue