mirror of
https://github.com/navidrome/navidrome.git
synced 2025-04-04 21:17:37 +03:00
feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)
* fix(server): more race conditions when updating artist/album from external sources
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): pass configfile option to child process
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): resume interrupted fullScans
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): remove old scanner code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): rename old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): move old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: tests
Signed-off-by: Deluan <deluan@navidrome.org>
* chore(deps): update Go to 1.23.4
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(test):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log level
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add config for scanner watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: children playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: replace `interface{}` with `any`
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlists with genres
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow any tags in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist names in playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlist's sort by tags
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use generic JSONArray for OS arrays
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use https in test
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add releaseTypes to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add recordLabels to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): rename JSONArray to Array
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): do not pre-populate smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement a simplified version of ArtistID3.
See https://github.com/opensubsonic/open-subsonic-api/discussions/120
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add contributors to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add albumArtists to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayArtist and displayAlbumArtist
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayComposer to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add roles to ArtistID3
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): use " • " separator for displayComposer
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): respect `PreferSortTags` config option
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize purging non-unused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: don't run 'refresh artist stats' concurrently with other transactions
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add Scanner.ScanOnStartup config option, default true
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: better json parsing error msg when importing NSPs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update album's imported_time when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: handle interrupted scans and full scans after migrations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `analyze` when migration requires a full rescan
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `PRAGMA optimize` at the end of the scan
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update artist's updated_at when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: handle multiple artists and roles in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): dim missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album missing logic
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error encoding in gob
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: separate warnings from errors
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: mark albums as missing if they were contained in a deleted folder
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: add participant names to media_file and album tables
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: use participations in criteria, instead of m2m relationship
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: rename participations to participants
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: albumartist role case
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): run scanner as an external process by default
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): show albumArtist names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): dim out missing albums
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: flaky test
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(server): scrobble buffer mapping. fix #3583
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: more participations renaming
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: send release_group_mbid to listenbrainz
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement OpenSubsonic explicitStatus field (#3597)
* feat: implement OpenSubsonic explicitStatus field
* fix(subsonic): fix failing snapshot tests
* refactor: create helper for setting explicitStatus
* fix: store smaller values for explicit-status on database
* test: ToAlbum explicitStatus
* refactor: rename explicitStatus helper function
---------
Co-authored-by: Deluan Quintão <deluan@navidrome.org>
* fix: handle album and track tags in the DB based on the mappings.yaml file
Signed-off-by: Deluan <deluan@navidrome.org>
* save similar artists as JSONB
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: getAlbumList byGenre
Signed-off-by: Deluan <deluan@navidrome.org>
* detect changes in PID configuration
Signed-off-by: Deluan <deluan@navidrome.org>
* set default album PID to legacy_pid
Signed-off-by: Deluan <deluan@navidrome.org>
* fix tests
Signed-off-by: Deluan <deluan@navidrome.org>
* fix SIGSEGV
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't lose album stars/ratings when migrating
Signed-off-by: Deluan <deluan@navidrome.org>
* store full PID conf in properties
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: keep album annotations when changing PID.Album config
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reassign album annotations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: use (display) albumArtist and add links to each artist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: not showing albums by albumartist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error msgs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: hide PID from Native API
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album cover art resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim participant names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reduce watcher log spam
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: panic when initializing the watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: various artists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't store empty lyrics in the DB
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused methods
Signed-off-by: Deluan <deluan@navidrome.org>
* drop full_text indexes, as they are not being used by SQLite
Signed-off-by: Deluan <deluan@navidrome.org>
* keep album created_at when upgrading
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null pointer
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album artwork cache
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't expose missing files in Subsonic API
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: searchable interface
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: filter out missing items from subsonic search
* fix: filter out missing items from playlists
* fix: filter out missing items from shares
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): add filter by artist role
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): only return albumartists in getIndexes and getArtists endpoints
Signed-off-by: Deluan <deluan@navidrome.org>
* sort roles alphabetically
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist playcounts
Signed-off-by: Deluan <deluan@navidrome.org>
* change default Album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix albumartist link when it does not match any albumartists values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix `Ignoring filter not whitelisted` (role) message
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim any names/titles being imported
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused genre code
Signed-off-by: Deluan <deluan@navidrome.org>
* serialize calls to Last.fm's getArtist
Signed-off-by: Deluan <deluan@navidrome.org>
xxx
Signed-off-by: Deluan <deluan@navidrome.org>
* add counters to genres
Signed-off-by: Deluan <deluan@navidrome.org>
* nit: fix migration `notice` message
Signed-off-by: Deluan <deluan@navidrome.org>
* optimize similar artists query
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: last.fm.getInfo when mbid does not exist
Signed-off-by: Deluan <deluan@navidrome.org>
* ui only show missing items for admins
Signed-off-by: Deluan <deluan@navidrome.org>
* don't allow interaction with missing items
Signed-off-by: Deluan <deluan@navidrome.org>
* Add Missing Files view (WIP)
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: merged tag_counts into tag table
Signed-off-by: Deluan <deluan@navidrome.org>
* add option to completely disable automatic scanner
Signed-off-by: Deluan <deluan@navidrome.org>
* add delete missing files functionality
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: playlists not showing for regular users
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce updateLastAccess frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce update player frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* add timeout when updating player
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dead code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix duplicated roles in stats
Signed-off-by: Deluan <deluan@navidrome.org>
* add `; ` to artist splitters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix stats query
Signed-off-by: Deluan <deluan@navidrome.org>
* more logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* add record label filter
Signed-off-by: Deluan <deluan@navidrome.org>
* add release type filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix purgeUnused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* add grouping filter to albums
Signed-off-by: Deluan <deluan@navidrome.org>
* allow any album tags to be used in as filters in the API
Signed-off-by: Deluan <deluan@navidrome.org>
* remove empty tags from album info
Signed-off-by: Deluan <deluan@navidrome.org>
* comments in the migration
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: Cannot read properties of undefined
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling (#3640)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove duplicated tag values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't ignore the taglib folder!
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: show track subtitle tag
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: show artists stats based on selected role
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: inspect
Signed-off-by: Deluan <deluan@navidrome.org>
* add media type to album info/filters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: change format of subtitle in the UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in Subsonic API and search
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in UI's player
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: split strings should be case-insensitive
Signed-off-by: Deluan <deluan@navidrome.org>
* disable ScanSchedule
Signed-off-by: Deluan <deluan@navidrome.org>
* increase default sessiontimeout
Signed-off-by: Deluan <deluan@navidrome.org>
* add sqlite command line tool to docker image
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: resources override
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* change migration to mark current artists as albumArtists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Allow filtering on multiple genres (#3679)
* feat(ui): Allow filtering on multiple genres
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
* add multi-genre filter in Album list
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
* add more multi-valued tag filters to Album and Song views
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): unselect missing files after removing
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): song filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix sharing tracks. fix #3687
Signed-off-by: Deluan <deluan@navidrome.org>
* use rowids when using search for sync (ex: Symfonium)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients for search
Signed-off-by: Deluan <deluan@navidrome.org>
* add libraryPath to Native API /songs endpoint
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add album version
Signed-off-by: Deluan <deluan@navidrome.org>
* made all tags lowercase as they are case-insensitive anyways.
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Show full paths, extended properties for album/song (#3691)
* feat(ui): Show full paths, extended properties for album/song
- uses library path + os separator + path
- show participants (album/song) and tags (song)
- make album/participant clickable in show info
* add source to path
* fix pathSeparator in UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix local artist artwork (#3695)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: parse vorbis performers
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: clean function into smaller functions
Signed-off-by: Deluan <deluan@navidrome.org>
* fix translations for en and pt
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow performers without instrument/subrole
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: metadata clean function again
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize split function
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: split function is now a method of TagConf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: humanize Artist total size
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version to album details
Signed-off-by: Deluan <deluan@navidrome.org>
* don't display album-level tags in SongInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* fix genre clicking in Album Page
Signed-off-by: Deluan <deluan@navidrome.org>
* don't use mbids in Last.fm api calls.
From 1337574018:
With MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo
{
artist: {
name: "Bee Gees",
mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810",
url: "https://www.last.fm/music/Bee+Gees",
}
```
Without MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo
{
artist: {
name: "Van Morrison",
mbid: "a41ac10f-0a56-4672-9161-b83f9b223559",
url: "https://www.last.fm/music/Van+Morrison",
}
```
Signed-off-by: Deluan <deluan@navidrome.org>
* better logging for when the artist folder is not found
Signed-off-by: Deluan <deluan@navidrome.org>
* fix various issues with artist image resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* hide "Additional Tags" header if there are none.
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify tag rendering
Signed-off-by: Deluan <deluan@navidrome.org>
* enhance logging for artist folder detection
Signed-off-by: Deluan <deluan@navidrome.org>
* make folderID consistent for relative and absolute folderPaths
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more folder paths scenarios
Signed-off-by: Deluan <deluan@navidrome.org>
* filter out other roles when SubsonicArtistParticipations = true
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Cannot read properties of undefined"
Signed-off-by: Deluan <deluan@navidrome.org>
* fix lyrics and comments being truncated (#3701)
* fix lyrics and comments being truncated
* specifically test for lyrics and comment length
* reorder assertions
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* fix(server): Expose library_path for playlist (#3705)
Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic)
* fix BFR on Windows (#3704)
* fix potential reflected cross-site scripting vulnerability
Signed-off-by: Deluan <deluan@navidrome.org>
* hack to make it work on Windows
* ignore windows executables
* try fixing the pipeline
Signed-off-by: Deluan <deluan@navidrome.org>
* allow MusicFolder in other drives
* move windows local drive logic to local storage implementation
---------
Signed-off-by: Deluan <deluan@navidrome.org>
* increase pagination sizes for missing files
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce level of "already scanning" watcher log message
Signed-off-by: Deluan <deluan@navidrome.org>
* only count folders with audio files in it
See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version and catalog number to search
Signed-off-by: Deluan <deluan@navidrome.org>
* add `organization` alias for `recordlabel`
Signed-off-by: Deluan <deluan@navidrome.org>
* remove mbid from Last.fm agent
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: support inspect in ui (#3726)
* inspect in ui
* address round 1
* add catalogNum to AlbumInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dependency on metadata_old (deprecated) package
Signed-off-by: Deluan <deluan@navidrome.org>
* add `RawTags` to model
Signed-off-by: Deluan <deluan@navidrome.org>
* support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698)
* parse standard roles, vorbis/m4a work for now
* fix djmixer
* working roles, use DJ-mix
* add performers to file
* map mbids
* add a few more tests
* add test
Signed-off-by: Deluan <deluan@navidrome.org>
* try to simplify the performers logic
Signed-off-by: Deluan <deluan@navidrome.org>
* stylistic changes
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* remove param mutation
Signed-off-by: Deluan <deluan@navidrome.org>
* run automated SQLite optimizations
Signed-off-by: Deluan <deluan@navidrome.org>
* fix playlists import/export on Windows
* fix import playlists
* fix export playlists
* better handling of Windows volumes
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more album ID reassignments
Signed-off-by: Deluan <deluan@navidrome.org>
* allow adding/overriding tags in the config file
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): Fix playlist track id, handle missing tracks better (#3734)
- Use `mediaFileId` instead of `id` for playlist tracks
- Only fetch if the file is not missing
- If extractor fails to get the file, also error (rather than panic)
* optimize DB after each scan.
Signed-off-by: Deluan <deluan@navidrome.org>
* remove sortable from AlbumSongs columns
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify query to get missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* mark Scanner.Extractor as deprecated
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Caio Cotts <caio@cotts.com.br>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão
GitHub
parent
46a963a02a
commit
c795bcfcf7
329 changed files with 16586 additions and 5852 deletions
|
|
@ -3,15 +3,15 @@ package persistence
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/deluan/rest"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
|
|
@ -19,180 +19,290 @@ type mediaFileRepository struct {
|
|||
sqlRepository
|
||||
}
|
||||
|
||||
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) *mediaFileRepository {
|
||||
type dbMediaFile struct {
|
||||
*model.MediaFile `structs:",flatten"`
|
||||
Participants string `structs:"-" json:"-"`
|
||||
Tags string `structs:"-" json:"-"`
|
||||
// These are necessary to map the correct names (rg_*) to the correct fields (RG*)
|
||||
// without using `db` struct tags in the model.MediaFile struct
|
||||
RgAlbumGain float64 `structs:"-" json:"-"`
|
||||
RgAlbumPeak float64 `structs:"-" json:"-"`
|
||||
RgTrackGain float64 `structs:"-" json:"-"`
|
||||
RgTrackPeak float64 `structs:"-" json:"-"`
|
||||
}
|
||||
|
||||
func (m *dbMediaFile) PostScan() error {
|
||||
m.RGTrackGain = m.RgTrackGain
|
||||
m.RGTrackPeak = m.RgTrackPeak
|
||||
m.RGAlbumGain = m.RgAlbumGain
|
||||
m.RGAlbumPeak = m.RgAlbumPeak
|
||||
var err error
|
||||
m.MediaFile.Participants, err = unmarshalParticipants(m.Participants)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing media_file from db: %w", err)
|
||||
}
|
||||
if m.Tags != "" {
|
||||
m.MediaFile.Tags, err = unmarshalTags(m.Tags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing media_file from db: %w", err)
|
||||
}
|
||||
m.Genre, m.Genres = m.MediaFile.Tags.ToGenres()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
|
||||
fullText := []string{m.FullTitle(), m.Album, m.Artist, m.AlbumArtist,
|
||||
m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle}
|
||||
fullText = append(fullText, m.MediaFile.Participants.AllNames()...)
|
||||
args["full_text"] = formatFullText(fullText...)
|
||||
args["tags"] = marshalTags(m.MediaFile.Tags)
|
||||
args["participants"] = marshalParticipants(m.MediaFile.Participants)
|
||||
return nil
|
||||
}
|
||||
|
||||
type dbMediaFiles []dbMediaFile
|
||||
|
||||
func (m dbMediaFiles) toModels() model.MediaFiles {
|
||||
return slice.Map(m, func(mf dbMediaFile) model.MediaFile { return *mf.MediaFile })
|
||||
}
|
||||
|
||||
func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFileRepository {
|
||||
r := &mediaFileRepository{}
|
||||
r.ctx = ctx
|
||||
r.db = db
|
||||
r.tableName = "media_file"
|
||||
r.registerModel(&model.MediaFile{}, map[string]filterFunc{
|
||||
"id": idFilter(r.tableName),
|
||||
"title": fullTextFilter,
|
||||
"starred": booleanFilter,
|
||||
"genre_id": eqFilter,
|
||||
})
|
||||
r.registerModel(&model.MediaFile{}, mediaFileFilter())
|
||||
r.setSortMappings(map[string]string{
|
||||
"title": "order_title",
|
||||
"artist": "order_artist_name, order_album_name, release_date, disc_number, track_number",
|
||||
"album": "order_album_name, release_date, disc_number, track_number, order_artist_name, title",
|
||||
"random": "random",
|
||||
"created_at": "media_file.created_at",
|
||||
"starred_at": "starred, starred_at",
|
||||
"title": "order_title",
|
||||
"artist": "order_artist_name, order_album_name, release_date, disc_number, track_number",
|
||||
"album_artist": "order_album_artist_name, order_album_name, release_date, disc_number, track_number",
|
||||
"album": "order_album_name, release_date, disc_number, track_number, order_artist_name, title",
|
||||
"random": "random",
|
||||
"created_at": "media_file.created_at",
|
||||
"starred_at": "starred, starred_at",
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
var mediaFileFilter = sync.OnceValue(func() map[string]filterFunc {
|
||||
filters := map[string]filterFunc{
|
||||
"id": idFilter("media_file"),
|
||||
"title": fullTextFilter("media_file"),
|
||||
"starred": booleanFilter,
|
||||
"genre_id": tagIDFilter,
|
||||
"missing": booleanFilter,
|
||||
}
|
||||
// Add all album tags as filters
|
||||
for tag := range model.TagMappings() {
|
||||
if _, exists := filters[string(tag)]; !exists {
|
||||
filters[string(tag)] = tagIDFilter
|
||||
}
|
||||
}
|
||||
return filters
|
||||
})
|
||||
|
||||
func (r *mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) {
|
||||
sql := r.newSelectWithAnnotation("media_file.id")
|
||||
sql = r.withGenres(sql) // Required for filtering by genre
|
||||
return r.count(sql, options...)
|
||||
query := r.newSelect()
|
||||
query = r.withAnnotation(query, "media_file.id")
|
||||
// BFR WithParticipants (for filtering by name)?
|
||||
return r.count(query, options...)
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Exists(id string) (bool, error) {
|
||||
return r.exists(Select().Where(Eq{"media_file.id": id}))
|
||||
return r.exists(Eq{"media_file.id": id})
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Put(m *model.MediaFile) error {
|
||||
m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist,
|
||||
m.SortTitle, m.SortAlbumName, m.SortArtistName, m.SortAlbumArtistName, m.DiscSubtitle)
|
||||
_, err := r.put(m.ID, m)
|
||||
m.CreatedAt = time.Now()
|
||||
id, err := r.putByMatch(Eq{"path": m.Path, "library_id": m.LibraryID}, m.ID, &dbMediaFile{MediaFile: m})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.updateGenres(m.ID, m.Genres)
|
||||
m.ID = id
|
||||
return r.updateParticipants(m.ID, m.Participants)
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) selectMediaFile(options ...model.QueryOptions) SelectBuilder {
|
||||
sql := r.newSelectWithAnnotation("media_file.id", options...).Columns("media_file.*")
|
||||
sql = r.withBookmark(sql, "media_file.id")
|
||||
if len(options) > 0 && options[0].Filters != nil {
|
||||
s, _, _ := options[0].Filters.ToSql()
|
||||
// If there's any reference of genre in the filter, joins with genre
|
||||
if strings.Contains(s, "genre") {
|
||||
sql = r.withGenres(sql)
|
||||
// If there's no filter on genre_id, group the results by media_file.id
|
||||
if !strings.Contains(s, "genre_id") {
|
||||
sql = sql.GroupBy("media_file.id")
|
||||
}
|
||||
}
|
||||
}
|
||||
return sql
|
||||
sql := r.newSelect(options...).Columns("media_file.*", "library.path as library_path").
|
||||
LeftJoin("library on media_file.library_id = library.id")
|
||||
sql = r.withAnnotation(sql, "media_file.id")
|
||||
return r.withBookmark(sql, "media_file.id")
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) {
|
||||
sel := r.selectMediaFile().Where(Eq{"media_file.id": id})
|
||||
var res model.MediaFiles
|
||||
if err := r.queryAll(sel, &res); err != nil {
|
||||
res, err := r.GetAll(model.QueryOptions{Filters: Eq{"media_file.id": id}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res) == 0 {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
err := loadAllGenres(r, res)
|
||||
return &res[0], err
|
||||
return &res[0], nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) GetWithParticipants(id string) (*model.MediaFile, error) {
|
||||
m, err := r.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Participants, err = r.getParticipants(m)
|
||||
return m, err
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) getParticipants(m *model.MediaFile) (model.Participants, error) {
|
||||
ar := NewArtistRepository(r.ctx, r.db)
|
||||
ids := m.Participants.AllIDs()
|
||||
artists, err := ar.GetAll(model.QueryOptions{Filters: Eq{"id": ids}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting participants: %w", err)
|
||||
}
|
||||
artistMap := slice.ToMap(artists, func(a model.Artist) (string, model.Artist) {
|
||||
return a.ID, a
|
||||
})
|
||||
p := m.Participants
|
||||
for role, artistList := range p {
|
||||
for idx, artist := range artistList {
|
||||
if a, ok := artistMap[artist.ID]; ok {
|
||||
p[role][idx].Artist = a
|
||||
}
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
|
||||
r.resetSeededRandom(options)
|
||||
sq := r.selectMediaFile(options...)
|
||||
res := model.MediaFiles{}
|
||||
var res dbMediaFiles
|
||||
err := r.queryAll(sq, &res, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = loadAllGenres(r, res)
|
||||
return res, err
|
||||
return res.toModels(), nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) GetCursor(options ...model.QueryOptions) (model.MediaFileCursor, error) {
|
||||
sq := r.selectMediaFile(options...)
|
||||
cursor, err := queryWithStableResults[dbMediaFile](r.sqlRepository, sq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func(yield func(model.MediaFile, error) bool) {
|
||||
for m, err := range cursor {
|
||||
if m.MediaFile == nil {
|
||||
yield(model.MediaFile{}, fmt.Errorf("unexpected nil mediafile: %v", m))
|
||||
return
|
||||
}
|
||||
if !yield(*m.MediaFile, err) || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, error) {
|
||||
sel := r.newSelect().Columns("*").Where(Eq{"path collate nocase": paths})
|
||||
var res model.MediaFiles
|
||||
var res dbMediaFiles
|
||||
if err := r.queryAll(sel, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func cleanPath(path string) string {
|
||||
path = filepath.Clean(path)
|
||||
if !strings.HasSuffix(path, string(os.PathSeparator)) {
|
||||
path += string(os.PathSeparator)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func pathStartsWith(path string) Eq {
|
||||
substr := fmt.Sprintf("substr(path, 1, %d)", utf8.RuneCountInString(path))
|
||||
return Eq{substr: path}
|
||||
}
|
||||
|
||||
// FindAllByPath only return mediafiles that are direct children of requested path
|
||||
func (r *mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, error) {
|
||||
// Query by path based on https://stackoverflow.com/a/13911906/653632
|
||||
path = cleanPath(path)
|
||||
pathLen := utf8.RuneCountInString(path)
|
||||
sel0 := r.newSelect().Columns("media_file.*", fmt.Sprintf("substr(path, %d) AS item", pathLen+2)).
|
||||
Where(pathStartsWith(path))
|
||||
sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
|
||||
Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
|
||||
|
||||
res := model.MediaFiles{}
|
||||
err := r.queryAll(sel, &res)
|
||||
return res, err
|
||||
}
|
||||
|
||||
// FindPathsRecursively returns a list of all subfolders of basePath, recursively
|
||||
func (r *mediaFileRepository) FindPathsRecursively(basePath string) ([]string, error) {
|
||||
path := cleanPath(basePath)
|
||||
// Query based on https://stackoverflow.com/a/38330814/653632
|
||||
sel := r.newSelect().Columns(fmt.Sprintf("distinct rtrim(path, replace(path, '%s', ''))", string(os.PathSeparator))).
|
||||
Where(pathStartsWith(path))
|
||||
var res []string
|
||||
err := r.queryAllSlice(sel, &res)
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) deleteNotInPath(basePath string) error {
|
||||
path := cleanPath(basePath)
|
||||
sel := Delete(r.tableName).Where(NotEq(pathStartsWith(path)))
|
||||
c, err := r.executeSQL(sel)
|
||||
if err == nil {
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Deleted dangling tracks", "totalDeleted", c)
|
||||
}
|
||||
}
|
||||
return err
|
||||
return res.toModels(), nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Delete(id string) error {
|
||||
return r.delete(Eq{"id": id})
|
||||
}
|
||||
|
||||
// DeleteByPath delete from the DB all mediafiles that are direct children of path
|
||||
func (r *mediaFileRepository) DeleteByPath(basePath string) (int64, error) {
|
||||
path := cleanPath(basePath)
|
||||
pathLen := utf8.RuneCountInString(path)
|
||||
del := Delete(r.tableName).
|
||||
Where(And{pathStartsWith(path),
|
||||
Eq{fmt.Sprintf("substr(path, %d) glob '*%s*'", pathLen+2, string(os.PathSeparator)): 0}})
|
||||
log.Debug(r.ctx, "Deleting mediafiles by path", "path", path)
|
||||
return r.executeSQL(del)
|
||||
func (r *mediaFileRepository) DeleteMissing(ids []string) error {
|
||||
user := loggedUser(r.ctx)
|
||||
if !user.IsAdmin {
|
||||
return rest.ErrPermissionDenied
|
||||
}
|
||||
return r.delete(
|
||||
And{
|
||||
Eq{"missing": true},
|
||||
Eq{"id": ids},
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) removeNonAlbumArtistIds() error {
|
||||
upd := Update(r.tableName).Set("artist_id", "").Where(notExists("artist", ConcatExpr("id = artist_id")))
|
||||
log.Debug(r.ctx, "Removing non-album artist_ids")
|
||||
_, err := r.executeSQL(upd)
|
||||
return err
|
||||
func (r *mediaFileRepository) MarkMissing(missing bool, mfs ...*model.MediaFile) error {
|
||||
ids := slice.SeqFunc(mfs, func(m *model.MediaFile) string { return m.ID })
|
||||
for chunk := range slice.CollectChunks(ids, 200) {
|
||||
upd := Update(r.tableName).
|
||||
Set("missing", missing).
|
||||
Set("updated_at", time.Now()).
|
||||
Where(Eq{"id": chunk})
|
||||
c, err := r.executeSQL(upd)
|
||||
if err != nil || c == 0 {
|
||||
log.Error(r.ctx, "Error setting mediafile missing flag", "ids", chunk, err)
|
||||
return err
|
||||
}
|
||||
log.Debug(r.ctx, "Marked missing mediafiles", "total", c, "ids", chunk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) {
|
||||
results := model.MediaFiles{}
|
||||
err := r.doSearch(q, offset, size, &results, "title")
|
||||
func (r *mediaFileRepository) MarkMissingByFolder(missing bool, folderIDs ...string) error {
|
||||
for chunk := range slices.Chunk(folderIDs, 200) {
|
||||
upd := Update(r.tableName).
|
||||
Set("missing", missing).
|
||||
Set("updated_at", time.Now()).
|
||||
Where(And{
|
||||
Eq{"folder_id": chunk},
|
||||
Eq{"missing": !missing},
|
||||
})
|
||||
c, err := r.executeSQL(upd)
|
||||
if err != nil {
|
||||
log.Error(r.ctx, "Error setting mediafile missing flag", "folderIDs", chunk, err)
|
||||
return err
|
||||
}
|
||||
log.Debug(r.ctx, "Marked missing mediafiles from missing folders", "total", c, "folders", chunk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetMissingAndMatching returns all mediafiles that are missing and their potential matches (comparing PIDs)
|
||||
// that were added/updated after the last scan started. The result is ordered by PID.
|
||||
// It does not need to load bookmarks, annotations and participnts, as they are not used by the scanner.
|
||||
func (r *mediaFileRepository) GetMissingAndMatching(libId int) (model.MediaFileCursor, error) {
|
||||
subQ := r.newSelect().Columns("pid").
|
||||
Where(And{
|
||||
Eq{"media_file.missing": true},
|
||||
Eq{"library_id": libId},
|
||||
})
|
||||
subQText, subQArgs, err := subQ.PlaceholderFormat(Question).ToSql()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = loadAllGenres(r, results)
|
||||
return results, err
|
||||
sel := r.newSelect().Columns("media_file.*", "library.path as library_path").
|
||||
LeftJoin("library on media_file.library_id = library.id").
|
||||
Where("pid in ("+subQText+")", subQArgs...).
|
||||
Where(Or{
|
||||
Eq{"missing": true},
|
||||
ConcatExpr("media_file.created_at > library.last_scan_started_at"),
|
||||
}).
|
||||
OrderBy("pid")
|
||||
cursor, err := queryWithStableResults[dbMediaFile](r.sqlRepository, sel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func(yield func(model.MediaFile, error) bool) {
|
||||
for m, err := range cursor {
|
||||
if !yield(*m.MediaFile, err) || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Search(q string, offset int, size int, includeMissing bool) (model.MediaFiles, error) {
|
||||
results := dbMediaFiles{}
|
||||
err := r.doSearch(r.selectMediaFile(), q, offset, size, includeMissing, &results, "title")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results.toModels(), err
|
||||
}
|
||||
|
||||
func (r *mediaFileRepository) Count(options ...rest.QueryOptions) (int64, error) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue