mirror of
https://github.com/navidrome/navidrome.git
synced 2025-04-03 20:47:35 +03:00
feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)
* fix(server): more race conditions when updating artist/album from external sources
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): pass configfile option to child process
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): resume interrupted fullScans
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): remove old scanner code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): rename old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): move old metadata package
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: tests
Signed-off-by: Deluan <deluan@navidrome.org>
* chore(deps): update Go to 1.23.4
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(test):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log level
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add config for scanner watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: children playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: replace `interface{}` with `any`
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlists with genres
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow any tags in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist names in playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: smart playlist's sort by tags
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use generic JSONArray for OS arrays
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): use https in test
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add releaseTypes to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add recordLabels to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic): rename JSONArray to Array
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to AlbumID3
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(scanner): do not pre-populate smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement a simplified version of ArtistID3.
See https://github.com/opensubsonic/open-subsonic-api/discussions/120
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add artists to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add contributors to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add albumArtists to mediafile Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayArtist and displayAlbumArtist
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add displayComposer to Child
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add roles to ArtistID3
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): use " • " separator for displayComposer
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(subsonic): respect `PreferSortTags` config option
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor(subsonic):
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize purging non-unused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: don't run 'refresh artist stats' concurrently with other transactions
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor:
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: log message
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add Scanner.ScanOnStartup config option, default true
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: better json parsing error msg when importing NSPs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update album's imported_time when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: handle interrupted scans and full scans after migrations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `analyze` when migration requires a full rescan
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: run `PRAGMA optimize` at the end of the scan
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't update artist's updated_at when updating external_metadata
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: handle multiple artists and roles in smart playlists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): dim missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album missing logic
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error encoding in gob
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: separate warnings from errors
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: mark albums as missing if they were contained in a deleted folder
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: add participant names to media_file and album tables
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: use participations in criteria, instead of m2m relationship
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: rename participations to participants
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add moods to album child
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: albumartist role case
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(scanner): run scanner as an external process by default
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): show albumArtist names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): dim out missing albums
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: flaky test
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(server): scrobble buffer mapping. fix #3583
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: more participations renaming
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: send release_group_mbid to listenbrainz
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): implement OpenSubsonic explicitStatus field (#3597)
* feat: implement OpenSubsonic explicitStatus field
* fix(subsonic): fix failing snapshot tests
* refactor: create helper for setting explicitStatus
* fix: store smaller values for explicit-status on database
* test: ToAlbum explicitStatus
* refactor: rename explicitStatus helper function
---------
Co-authored-by: Deluan Quintão <deluan@navidrome.org>
* fix: handle album and track tags in the DB based on the mappings.yaml file
Signed-off-by: Deluan <deluan@navidrome.org>
* save similar artists as JSONB
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: getAlbumList byGenre
Signed-off-by: Deluan <deluan@navidrome.org>
* detect changes in PID configuration
Signed-off-by: Deluan <deluan@navidrome.org>
* set default album PID to legacy_pid
Signed-off-by: Deluan <deluan@navidrome.org>
* fix tests
Signed-off-by: Deluan <deluan@navidrome.org>
* fix SIGSEGV
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't lose album stars/ratings when migrating
Signed-off-by: Deluan <deluan@navidrome.org>
* store full PID conf in properties
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: keep album annotations when changing PID.Album config
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reassign album annotations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: use (display) albumArtist and add links to each artist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: not showing albums by albumartist
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: error msgs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: hide PID from Native API
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album cover art resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim participant names
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: reduce watcher log spam
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: panic when initializing the watcher
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: various artists
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't store empty lyrics in the DB
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused methods
Signed-off-by: Deluan <deluan@navidrome.org>
* drop full_text indexes, as they are not being used by SQLite
Signed-off-by: Deluan <deluan@navidrome.org>
* keep album created_at when upgrading
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): null pointer
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album artwork cache
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't expose missing files in Subsonic API
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: searchable interface
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: filter out missing items from subsonic search
* fix: filter out missing items from playlists
* fix: filter out missing items from shares
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): add filter by artist role
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): only return albumartists in getIndexes and getArtists endpoints
Signed-off-by: Deluan <deluan@navidrome.org>
* sort roles alphabetically
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: artist playcounts
Signed-off-by: Deluan <deluan@navidrome.org>
* change default Album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix albumartist link when it does not match any albumartists values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix `Ignoring filter not whitelisted` (role) message
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: trim any names/titles being imported
Signed-off-by: Deluan <deluan@navidrome.org>
* remove unused genre code
Signed-off-by: Deluan <deluan@navidrome.org>
* serialize calls to Last.fm's getArtist
Signed-off-by: Deluan <deluan@navidrome.org>
xxx
Signed-off-by: Deluan <deluan@navidrome.org>
* add counters to genres
Signed-off-by: Deluan <deluan@navidrome.org>
* nit: fix migration `notice` message
Signed-off-by: Deluan <deluan@navidrome.org>
* optimize similar artists query
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: last.fm.getInfo when mbid does not exist
Signed-off-by: Deluan <deluan@navidrome.org>
* ui only show missing items for admins
Signed-off-by: Deluan <deluan@navidrome.org>
* don't allow interaction with missing items
Signed-off-by: Deluan <deluan@navidrome.org>
* Add Missing Files view (WIP)
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: merged tag_counts into tag table
Signed-off-by: Deluan <deluan@navidrome.org>
* add option to completely disable automatic scanner
Signed-off-by: Deluan <deluan@navidrome.org>
* add delete missing files functionality
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: playlists not showing for regular users
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce updateLastAccess frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce update player frequency to once every minute
Signed-off-by: Deluan <deluan@navidrome.org>
* add timeout when updating player
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dead code
Signed-off-by: Deluan <deluan@navidrome.org>
* fix duplicated roles in stats
Signed-off-by: Deluan <deluan@navidrome.org>
* add `; ` to artist splitters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix stats query
Signed-off-by: Deluan <deluan@navidrome.org>
* more logs
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP
Signed-off-by: Deluan <deluan@navidrome.org>
* add record label filter
Signed-off-by: Deluan <deluan@navidrome.org>
* add release type filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix purgeUnused tags
Signed-off-by: Deluan <deluan@navidrome.org>
* add grouping filter to albums
Signed-off-by: Deluan <deluan@navidrome.org>
* allow any album tags to be used in as filters in the API
Signed-off-by: Deluan <deluan@navidrome.org>
* remove empty tags from album info
Signed-off-by: Deluan <deluan@navidrome.org>
* comments in the migration
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: Cannot read properties of undefined
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: listenbrainz scrobbling (#3640)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: remove duplicated tag values
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: don't ignore the taglib folder!
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: show track subtitle tag
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: show artists stats based on selected role
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: inspect
Signed-off-by: Deluan <deluan@navidrome.org>
* add media type to album info/filters
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: change format of subtitle in the UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in Subsonic API and search
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: subtitle in UI's player
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: split strings should be case-insensitive
Signed-off-by: Deluan <deluan@navidrome.org>
* disable ScanSchedule
Signed-off-by: Deluan <deluan@navidrome.org>
* increase default sessiontimeout
Signed-off-by: Deluan <deluan@navidrome.org>
* add sqlite command line tool to docker image
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: resources override
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: album PID conf
Signed-off-by: Deluan <deluan@navidrome.org>
* change migration to mark current artists as albumArtists
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Allow filtering on multiple genres (#3679)
* feat(ui): Allow filtering on multiple genres
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
* add multi-genre filter in Album list
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
* add more multi-valued tag filters to Album and Song views
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): unselect missing files after removing
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): song filter
Signed-off-by: Deluan <deluan@navidrome.org>
* fix sharing tracks. fix #3687
Signed-off-by: Deluan <deluan@navidrome.org>
* use rowids when using search for sync (ex: Symfonium)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Report Real Paths" option for subsonic clients for search
Signed-off-by: Deluan <deluan@navidrome.org>
* add libraryPath to Native API /songs endpoint
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(subsonic): add album version
Signed-off-by: Deluan <deluan@navidrome.org>
* made all tags lowercase as they are case-insensitive anyways.
Signed-off-by: Deluan <deluan@navidrome.org>
* feat(ui): Show full paths, extended properties for album/song (#3691)
* feat(ui): Show full paths, extended properties for album/song
- uses library path + os separator + path
- show participants (album/song) and tags (song)
- make album/participant clickable in show info
* add source to path
* fix pathSeparator in UI
Signed-off-by: Deluan <deluan@navidrome.org>
* fix local artist artwork (#3695)
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: parse vorbis performers
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: clean function into smaller functions
Signed-off-by: Deluan <deluan@navidrome.org>
* fix translations for en and pt
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* add trace log to show annotations reassignment
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: allow performers without instrument/subrole
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: metadata clean function again
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: optimize split function
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: split function is now a method of TagConf
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: humanize Artist total size
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version to album details
Signed-off-by: Deluan <deluan@navidrome.org>
* don't display album-level tags in SongInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* fix genre clicking in Album Page
Signed-off-by: Deluan <deluan@navidrome.org>
* don't use mbids in Last.fm api calls.
From 1337574018:
With MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo
{
artist: {
name: "Bee Gees",
mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810",
url: "https://www.last.fm/music/Bee+Gees",
}
```
Without MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo
{
artist: {
name: "Van Morrison",
mbid: "a41ac10f-0a56-4672-9161-b83f9b223559",
url: "https://www.last.fm/music/Van+Morrison",
}
```
Signed-off-by: Deluan <deluan@navidrome.org>
* better logging for when the artist folder is not found
Signed-off-by: Deluan <deluan@navidrome.org>
* fix various issues with artist image resolution
Signed-off-by: Deluan <deluan@navidrome.org>
* hide "Additional Tags" header if there are none.
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify tag rendering
Signed-off-by: Deluan <deluan@navidrome.org>
* enhance logging for artist folder detection
Signed-off-by: Deluan <deluan@navidrome.org>
* make folderID consistent for relative and absolute folderPaths
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more folder paths scenarios
Signed-off-by: Deluan <deluan@navidrome.org>
* filter out other roles when SubsonicArtistParticipations = true
Signed-off-by: Deluan <deluan@navidrome.org>
* fix "Cannot read properties of undefined"
Signed-off-by: Deluan <deluan@navidrome.org>
* fix lyrics and comments being truncated (#3701)
* fix lyrics and comments being truncated
* specifically test for lyrics and comment length
* reorder assertions
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* fix(server): Expose library_path for playlist (#3705)
Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic)
* fix BFR on Windows (#3704)
* fix potential reflected cross-site scripting vulnerability
Signed-off-by: Deluan <deluan@navidrome.org>
* hack to make it work on Windows
* ignore windows executables
* try fixing the pipeline
Signed-off-by: Deluan <deluan@navidrome.org>
* allow MusicFolder in other drives
* move windows local drive logic to local storage implementation
---------
Signed-off-by: Deluan <deluan@navidrome.org>
* increase pagination sizes for missing files
Signed-off-by: Deluan <deluan@navidrome.org>
* reduce level of "already scanning" watcher log message
Signed-off-by: Deluan <deluan@navidrome.org>
* only count folders with audio files in it
See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930
Signed-off-by: Deluan <deluan@navidrome.org>
* add album version and catalog number to search
Signed-off-by: Deluan <deluan@navidrome.org>
* add `organization` alias for `recordlabel`
Signed-off-by: Deluan <deluan@navidrome.org>
* remove mbid from Last.fm agent
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: support inspect in ui (#3726)
* inspect in ui
* address round 1
* add catalogNum to AlbumInfo
Signed-off-by: Deluan <deluan@navidrome.org>
* remove dependency on metadata_old (deprecated) package
Signed-off-by: Deluan <deluan@navidrome.org>
* add `RawTags` to model
Signed-off-by: Deluan <deluan@navidrome.org>
* support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698)
* parse standard roles, vorbis/m4a work for now
* fix djmixer
* working roles, use DJ-mix
* add performers to file
* map mbids
* add a few more tests
* add test
Signed-off-by: Deluan <deluan@navidrome.org>
* try to simplify the performers logic
Signed-off-by: Deluan <deluan@navidrome.org>
* stylistic changes
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>
* remove param mutation
Signed-off-by: Deluan <deluan@navidrome.org>
* run automated SQLite optimizations
Signed-off-by: Deluan <deluan@navidrome.org>
* fix playlists import/export on Windows
* fix import playlists
* fix export playlists
* better handling of Windows volumes
Signed-off-by: Deluan <deluan@navidrome.org>
* handle more album ID reassignments
Signed-off-by: Deluan <deluan@navidrome.org>
* allow adding/overriding tags in the config file
Signed-off-by: Deluan <deluan@navidrome.org>
* fix(ui): Fix playlist track id, handle missing tracks better (#3734)
- Use `mediaFileId` instead of `id` for playlist tracks
- Only fetch if the file is not missing
- If extractor fails to get the file, also error (rather than panic)
* optimize DB after each scan.
Signed-off-by: Deluan <deluan@navidrome.org>
* remove sortable from AlbumSongs columns
Signed-off-by: Deluan <deluan@navidrome.org>
* simplify query to get missing tracks
Signed-off-by: Deluan <deluan@navidrome.org>
* mark Scanner.Extractor as deprecated
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Caio Cotts <caio@cotts.com.br>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão
GitHub
parent
46a963a02a
commit
c795bcfcf7
329 changed files with 16586 additions and 5852 deletions
|
|
@ -1,47 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/cache"
|
||||
"github.com/navidrome/navidrome/utils/singleton"
|
||||
)
|
||||
|
||||
func newCachedGenreRepository(ctx context.Context, repo model.GenreRepository) model.GenreRepository {
|
||||
return singleton.GetInstance(func() *cachedGenreRepo {
|
||||
r := &cachedGenreRepo{
|
||||
GenreRepository: repo,
|
||||
ctx: ctx,
|
||||
}
|
||||
genres, err := repo.GetAll()
|
||||
|
||||
if err != nil {
|
||||
log.Error(ctx, "Could not load genres from DB", err)
|
||||
panic(err)
|
||||
}
|
||||
r.cache = cache.NewSimpleCache[string, string]()
|
||||
for _, g := range genres {
|
||||
_ = r.cache.Add(strings.ToLower(g.Name), g.ID)
|
||||
}
|
||||
return r
|
||||
})
|
||||
}
|
||||
|
||||
type cachedGenreRepo struct {
|
||||
model.GenreRepository
|
||||
cache cache.SimpleCache[string, string]
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (r *cachedGenreRepo) Put(g *model.Genre) error {
|
||||
id, err := r.cache.GetWithLoader(strings.ToLower(g.Name), func(key string) (string, time.Duration, error) {
|
||||
err := r.GenreRepository.Put(g)
|
||||
return g.ID, 24 * time.Hour, err
|
||||
})
|
||||
g.ID = id
|
||||
return err
|
||||
}
|
||||
260
scanner/controller.go
Normal file
260
scanner/controller.go
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/auth"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
. "github.com/navidrome/navidrome/utils/gg"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAlreadyScanning = errors.New("already scanning")
|
||||
)
|
||||
|
||||
type Scanner interface {
|
||||
// ScanAll starts a full scan of the music library. This is a blocking operation.
|
||||
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
|
||||
Status(context.Context) (*StatusInfo, error)
|
||||
}
|
||||
|
||||
type StatusInfo struct {
|
||||
Scanning bool
|
||||
LastScan time.Time
|
||||
Count uint32
|
||||
FolderCount uint32
|
||||
}
|
||||
|
||||
func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker,
|
||||
pls core.Playlists, m metrics.Metrics) Scanner {
|
||||
c := &controller{
|
||||
rootCtx: rootCtx,
|
||||
ds: ds,
|
||||
cw: cw,
|
||||
broker: broker,
|
||||
pls: pls,
|
||||
metrics: m,
|
||||
}
|
||||
if !conf.Server.DevExternalScanner {
|
||||
c.limiter = P(rate.Sometimes{Interval: conf.Server.DevActivityPanelUpdateRate})
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (s *controller) getScanner() scanner {
|
||||
if conf.Server.DevExternalScanner {
|
||||
return &scannerExternal{}
|
||||
}
|
||||
return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls, metrics: s.metrics}
|
||||
}
|
||||
|
||||
// CallScan starts an in-process scan of the music library.
|
||||
// This is meant to be called from the command line (see cmd/scan.go).
|
||||
func CallScan(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, pls core.Playlists,
|
||||
metrics metrics.Metrics, fullScan bool) (<-chan *ProgressInfo, error) {
|
||||
release, err := lockScan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
ctx = auth.WithAdminUser(ctx, ds)
|
||||
progress := make(chan *ProgressInfo, 100)
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := &scannerImpl{ds: ds, cw: cw, pls: pls, metrics: metrics}
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
}()
|
||||
return progress, nil
|
||||
}
|
||||
|
||||
func IsScanning() bool {
|
||||
return running.Load()
|
||||
}
|
||||
|
||||
type ProgressInfo struct {
|
||||
LibID int
|
||||
FileCount uint32
|
||||
Path string
|
||||
Phase string
|
||||
ChangesDetected bool
|
||||
Warning string
|
||||
Error string
|
||||
}
|
||||
|
||||
type scanner interface {
|
||||
scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo)
|
||||
// BFR: scanFolders(ctx context.Context, lib model.Lib, folders []string, progress chan<- *ScannerStatus)
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
rootCtx context.Context
|
||||
ds model.DataStore
|
||||
cw artwork.CacheWarmer
|
||||
broker events.Broker
|
||||
metrics metrics.Metrics
|
||||
pls core.Playlists
|
||||
limiter *rate.Sometimes
|
||||
count atomic.Uint32
|
||||
folderCount atomic.Uint32
|
||||
changesDetected bool
|
||||
}
|
||||
|
||||
func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
|
||||
lib, err := s.ds.Library(ctx).Get(1) //TODO Multi-library
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting library: %w", err)
|
||||
}
|
||||
if running.Load() {
|
||||
status := &StatusInfo{
|
||||
Scanning: true,
|
||||
LastScan: lib.LastScanAt,
|
||||
Count: s.count.Load(),
|
||||
FolderCount: s.folderCount.Load(),
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
count, folderCount, err := s.getCounters(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting library stats: %w", err)
|
||||
}
|
||||
return &StatusInfo{
|
||||
Scanning: false,
|
||||
LastScan: lib.LastScanAt,
|
||||
Count: uint32(count),
|
||||
FolderCount: uint32(folderCount),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *controller) getCounters(ctx context.Context) (int64, int64, error) {
|
||||
count, err := s.ds.MediaFile(ctx).CountAll()
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("media file count: %w", err)
|
||||
}
|
||||
folderCount, err := s.ds.Folder(ctx).CountAll(
|
||||
model.QueryOptions{
|
||||
Filters: squirrel.And{
|
||||
squirrel.Gt{"num_audio_files": 0},
|
||||
squirrel.Eq{"missing": false},
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("folder count: %w", err)
|
||||
}
|
||||
return count, folderCount, nil
|
||||
}
|
||||
|
||||
func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) {
|
||||
release, err := lockScan(requestCtx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
// Prepare the context for the scan
|
||||
ctx := request.AddValues(s.rootCtx, requestCtx)
|
||||
ctx = events.BroadcastToAll(ctx)
|
||||
ctx = auth.WithAdminUser(ctx, s.ds)
|
||||
|
||||
// Send the initial scan status event
|
||||
s.sendMessage(ctx, &events.ScanStatus{Scanning: true, Count: 0, FolderCount: 0})
|
||||
progress := make(chan *ProgressInfo, 100)
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := s.getScanner()
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
}()
|
||||
|
||||
// Wait for the scan to finish, sending progress events to all connected clients
|
||||
scanWarnings, scanError := s.trackProgress(ctx, progress)
|
||||
for _, w := range scanWarnings {
|
||||
log.Warn(ctx, fmt.Sprintf("Scan warning: %s", w))
|
||||
}
|
||||
// If changes were detected, send a refresh event to all clients
|
||||
if s.changesDetected {
|
||||
log.Debug(ctx, "Library changes imported. Sending refresh event")
|
||||
s.broker.SendMessage(ctx, &events.RefreshResource{})
|
||||
}
|
||||
// Send the final scan status event, with totals
|
||||
if count, folderCount, err := s.getCounters(ctx); err != nil {
|
||||
return scanWarnings, err
|
||||
} else {
|
||||
s.sendMessage(ctx, &events.ScanStatus{
|
||||
Scanning: false,
|
||||
Count: count,
|
||||
FolderCount: folderCount,
|
||||
})
|
||||
}
|
||||
return scanWarnings, scanError
|
||||
}
|
||||
|
||||
// This is a global variable that is used to prevent multiple scans from running at the same time.
|
||||
// "There can be only one" - https://youtu.be/sqcLjcSloXs?si=VlsjEOjTJZ68zIyg
|
||||
var running atomic.Bool
|
||||
|
||||
func lockScan(ctx context.Context) (func(), error) {
|
||||
if !running.CompareAndSwap(false, true) {
|
||||
log.Debug(ctx, "Scanner already running, ignoring request")
|
||||
return func() {}, ErrAlreadyScanning
|
||||
}
|
||||
return func() {
|
||||
running.Store(false)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *controller) trackProgress(ctx context.Context, progress <-chan *ProgressInfo) ([]string, error) {
|
||||
s.count.Store(0)
|
||||
s.folderCount.Store(0)
|
||||
s.changesDetected = false
|
||||
|
||||
var warnings []string
|
||||
var errs []error
|
||||
for p := range pl.ReadOrDone(ctx, progress) {
|
||||
if p.Error != "" {
|
||||
errs = append(errs, errors.New(p.Error))
|
||||
continue
|
||||
}
|
||||
if p.Warning != "" {
|
||||
warnings = append(warnings, p.Warning)
|
||||
continue
|
||||
}
|
||||
if p.ChangesDetected {
|
||||
s.changesDetected = true
|
||||
continue
|
||||
}
|
||||
s.count.Add(p.FileCount)
|
||||
if p.FileCount > 0 {
|
||||
s.folderCount.Add(1)
|
||||
}
|
||||
status := &events.ScanStatus{
|
||||
Scanning: true,
|
||||
Count: int64(s.count.Load()),
|
||||
FolderCount: int64(s.folderCount.Load()),
|
||||
}
|
||||
if s.limiter != nil {
|
||||
s.limiter.Do(func() { s.sendMessage(ctx, status) })
|
||||
} else {
|
||||
s.sendMessage(ctx, status)
|
||||
}
|
||||
}
|
||||
return warnings, errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (s *controller) sendMessage(ctx context.Context, status *events.ScanStatus) {
|
||||
s.broker.SendMessage(ctx, status)
|
||||
}
|
||||
76
scanner/external.go
Normal file
76
scanner/external.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
. "github.com/navidrome/navidrome/utils/gg"
|
||||
)
|
||||
|
||||
// scannerExternal is a scanner that runs an external process to do the scanning. It is used to avoid
|
||||
// memory leaks or retention in the main process, as the scanner can consume a lot of memory. The
|
||||
// external process will be spawned with the same executable as the current process, and will run
|
||||
// the "scan" command with the "--subprocess" flag.
|
||||
//
|
||||
// The external process will send progress updates to the main process through its STDOUT, and the main
|
||||
// process will forward them to the caller.
|
||||
type scannerExternal struct{}
|
||||
|
||||
func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)}
|
||||
return
|
||||
}
|
||||
log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe)
|
||||
cmd := exec.CommandContext(ctx, exe, "scan",
|
||||
"--nobanner", "--subprocess",
|
||||
"--configfile", conf.Server.ConfigFile,
|
||||
If(fullScan, "--full", ""))
|
||||
|
||||
in, out := io.Pipe()
|
||||
defer in.Close()
|
||||
defer out.Close()
|
||||
cmd.Stdout = out
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to start scanner process: %s", err)}
|
||||
return
|
||||
}
|
||||
go s.wait(cmd, out)
|
||||
|
||||
decoder := gob.NewDecoder(in)
|
||||
for {
|
||||
var p ProgressInfo
|
||||
if err := decoder.Decode(&p); err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to read status from scanner: %s", err)}
|
||||
}
|
||||
break
|
||||
}
|
||||
progress <- &p
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scannerExternal) wait(cmd *exec.Cmd, out *io.PipeWriter) {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) {
|
||||
_ = out.CloseWithError(fmt.Errorf("%s exited with non-zero status code: %w", cmd, exitErr))
|
||||
} else {
|
||||
_ = out.CloseWithError(fmt.Errorf("waiting %s cmd: %w", cmd, err))
|
||||
}
|
||||
return
|
||||
}
|
||||
_ = out.Close()
|
||||
}
|
||||
|
||||
var _ scanner = (*scannerExternal)(nil)
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
"github.com/navidrome/navidrome/utils/str"
|
||||
)
|
||||
|
||||
type MediaFileMapper struct {
|
||||
rootFolder string
|
||||
genres model.GenreRepository
|
||||
}
|
||||
|
||||
func NewMediaFileMapper(rootFolder string, genres model.GenreRepository) *MediaFileMapper {
|
||||
return &MediaFileMapper{
|
||||
rootFolder: rootFolder,
|
||||
genres: genres,
|
||||
}
|
||||
}
|
||||
|
||||
// TODO Move most of these mapping functions to setters in the model.MediaFile
|
||||
func (s MediaFileMapper) ToMediaFile(md metadata.Tags) model.MediaFile {
|
||||
mf := &model.MediaFile{}
|
||||
mf.ID = s.trackID(md)
|
||||
mf.Year, mf.Date, mf.OriginalYear, mf.OriginalDate, mf.ReleaseYear, mf.ReleaseDate = s.mapDates(md)
|
||||
mf.Title = s.mapTrackTitle(md)
|
||||
mf.Album = md.Album()
|
||||
mf.AlbumID = s.albumID(md, mf.ReleaseDate)
|
||||
mf.Album = s.mapAlbumName(md)
|
||||
mf.ArtistID = s.artistID(md)
|
||||
mf.Artist = s.mapArtistName(md)
|
||||
mf.AlbumArtistID = s.albumArtistID(md)
|
||||
mf.AlbumArtist = s.mapAlbumArtistName(md)
|
||||
mf.Genre, mf.Genres = s.mapGenres(md.Genres())
|
||||
mf.Compilation = md.Compilation()
|
||||
mf.TrackNumber, _ = md.TrackNumber()
|
||||
mf.DiscNumber, _ = md.DiscNumber()
|
||||
mf.DiscSubtitle = md.DiscSubtitle()
|
||||
mf.Duration = md.Duration()
|
||||
mf.BitRate = md.BitRate()
|
||||
mf.SampleRate = md.SampleRate()
|
||||
mf.Channels = md.Channels()
|
||||
mf.Path = md.FilePath()
|
||||
mf.Suffix = md.Suffix()
|
||||
mf.Size = md.Size()
|
||||
mf.HasCoverArt = md.HasPicture()
|
||||
mf.SortTitle = md.SortTitle()
|
||||
mf.SortAlbumName = md.SortAlbum()
|
||||
mf.SortArtistName = md.SortArtist()
|
||||
mf.SortAlbumArtistName = md.SortAlbumArtist()
|
||||
mf.OrderTitle = str.SanitizeFieldForSorting(mf.Title)
|
||||
mf.OrderAlbumName = str.SanitizeFieldForSortingNoArticle(mf.Album)
|
||||
mf.OrderArtistName = str.SanitizeFieldForSortingNoArticle(mf.Artist)
|
||||
mf.OrderAlbumArtistName = str.SanitizeFieldForSortingNoArticle(mf.AlbumArtist)
|
||||
mf.CatalogNum = md.CatalogNum()
|
||||
mf.MbzRecordingID = md.MbzRecordingID()
|
||||
mf.MbzReleaseTrackID = md.MbzReleaseTrackID()
|
||||
mf.MbzAlbumID = md.MbzAlbumID()
|
||||
mf.MbzArtistID = md.MbzArtistID()
|
||||
mf.MbzAlbumArtistID = md.MbzAlbumArtistID()
|
||||
mf.MbzAlbumType = md.MbzAlbumType()
|
||||
mf.MbzAlbumComment = md.MbzAlbumComment()
|
||||
mf.RgAlbumGain = md.RGAlbumGain()
|
||||
mf.RgAlbumPeak = md.RGAlbumPeak()
|
||||
mf.RgTrackGain = md.RGTrackGain()
|
||||
mf.RgTrackPeak = md.RGTrackPeak()
|
||||
mf.Comment = str.SanitizeText(md.Comment())
|
||||
mf.Lyrics = md.Lyrics()
|
||||
mf.Bpm = md.Bpm()
|
||||
mf.CreatedAt = md.BirthTime()
|
||||
mf.UpdatedAt = md.ModificationTime()
|
||||
|
||||
return *mf
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapTrackTitle(md metadata.Tags) string {
|
||||
if md.Title() == "" {
|
||||
s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
|
||||
e := filepath.Ext(s)
|
||||
return strings.TrimSuffix(s, e)
|
||||
}
|
||||
return md.Title()
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapAlbumArtistName(md metadata.Tags) string {
|
||||
switch {
|
||||
case md.AlbumArtist() != "":
|
||||
return md.AlbumArtist()
|
||||
case md.Compilation():
|
||||
return consts.VariousArtists
|
||||
case md.Artist() != "":
|
||||
return md.Artist()
|
||||
default:
|
||||
return consts.UnknownArtist
|
||||
}
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapArtistName(md metadata.Tags) string {
|
||||
if md.Artist() != "" {
|
||||
return md.Artist()
|
||||
}
|
||||
return consts.UnknownArtist
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapAlbumName(md metadata.Tags) string {
|
||||
name := md.Album()
|
||||
if name == "" {
|
||||
return consts.UnknownAlbum
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) trackID(md metadata.Tags) string {
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) albumID(md metadata.Tags, releaseDate string) string {
|
||||
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
|
||||
if !conf.Server.Scanner.GroupAlbumReleases {
|
||||
if len(releaseDate) != 0 {
|
||||
albumPath = fmt.Sprintf("%s\\%s", albumPath, releaseDate)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) artistID(md metadata.Tags) string {
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) albumArtistID(md metadata.Tags) string {
|
||||
return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapGenres(genres []string) (string, model.Genres) {
|
||||
var result model.Genres
|
||||
unique := map[string]struct{}{}
|
||||
all := make([]string, 0, len(genres)*2)
|
||||
for i := range genres {
|
||||
gs := strings.FieldsFunc(genres[i], func(r rune) bool {
|
||||
return strings.ContainsRune(conf.Server.Scanner.GenreSeparators, r)
|
||||
})
|
||||
for j := range gs {
|
||||
g := strings.TrimSpace(gs[j])
|
||||
key := strings.ToLower(g)
|
||||
if _, ok := unique[key]; ok {
|
||||
continue
|
||||
}
|
||||
all = append(all, g)
|
||||
unique[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, g := range all {
|
||||
genre := model.Genre{Name: g}
|
||||
_ = s.genres.Put(&genre)
|
||||
result = append(result, genre)
|
||||
}
|
||||
if len(result) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return result[0].Name, result
|
||||
}
|
||||
|
||||
func (s MediaFileMapper) mapDates(md metadata.Tags) (year int, date string,
|
||||
originalYear int, originalDate string,
|
||||
releaseYear int, releaseDate string) {
|
||||
// Start with defaults
|
||||
year, date = md.Date()
|
||||
originalYear, originalDate = md.OriginalDate()
|
||||
releaseYear, releaseDate = md.ReleaseDate()
|
||||
|
||||
// MusicBrainz Picard writes the Release Date of an album to the Date tag, and leaves the Release Date tag empty
|
||||
taggedLikePicard := (originalYear != 0) &&
|
||||
(releaseYear == 0) &&
|
||||
(year >= originalYear)
|
||||
if taggedLikePicard {
|
||||
return originalYear, originalDate, originalYear, originalDate, year, date
|
||||
}
|
||||
// when there's no Date, first fall back to Original Date, then to Release Date.
|
||||
if year == 0 {
|
||||
if originalYear > 0 {
|
||||
year, date = originalYear, originalDate
|
||||
} else {
|
||||
year, date = releaseYear, releaseDate
|
||||
}
|
||||
}
|
||||
return year, date, originalYear, originalDate, releaseYear, releaseDate
|
||||
}
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("mapping", func() {
|
||||
Describe("MediaFileMapper", func() {
|
||||
var mapper *MediaFileMapper
|
||||
Describe("mapTrackTitle", func() {
|
||||
BeforeEach(func() {
|
||||
mapper = NewMediaFileMapper("/music", nil)
|
||||
})
|
||||
It("returns the Title when it is available", func() {
|
||||
md := metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{"title": []string{"This is not a love song"}})
|
||||
Expect(mapper.mapTrackTitle(md)).To(Equal("This is not a love song"))
|
||||
})
|
||||
It("returns the filename if Title is not set", func() {
|
||||
md := metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{})
|
||||
Expect(mapper.mapTrackTitle(md)).To(Equal("artist/album01/Song"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("mapGenres", func() {
|
||||
var gr model.GenreRepository
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
ds := &tests.MockDataStore{}
|
||||
gr = ds.Genre(ctx)
|
||||
gr = newCachedGenreRepository(ctx, gr)
|
||||
mapper = NewMediaFileMapper("/", gr)
|
||||
})
|
||||
|
||||
It("returns empty if no genres are available", func() {
|
||||
g, gs := mapper.mapGenres(nil)
|
||||
Expect(g).To(BeEmpty())
|
||||
Expect(gs).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns genres", func() {
|
||||
g, gs := mapper.mapGenres([]string{"Rock", "Electronic"})
|
||||
Expect(g).To(Equal("Rock"))
|
||||
Expect(gs).To(HaveLen(2))
|
||||
Expect(gs[0].Name).To(Equal("Rock"))
|
||||
Expect(gs[1].Name).To(Equal("Electronic"))
|
||||
})
|
||||
|
||||
It("parses multi-valued genres", func() {
|
||||
g, gs := mapper.mapGenres([]string{"Rock;Dance", "Electronic", "Rock"})
|
||||
Expect(g).To(Equal("Rock"))
|
||||
Expect(gs).To(HaveLen(3))
|
||||
Expect(gs[0].Name).To(Equal("Rock"))
|
||||
Expect(gs[1].Name).To(Equal("Dance"))
|
||||
Expect(gs[2].Name).To(Equal("Electronic"))
|
||||
})
|
||||
It("trims genres names", func() {
|
||||
_, gs := mapper.mapGenres([]string{"Rock ; Dance", " Electronic "})
|
||||
Expect(gs).To(HaveLen(3))
|
||||
Expect(gs[0].Name).To(Equal("Rock"))
|
||||
Expect(gs[1].Name).To(Equal("Dance"))
|
||||
Expect(gs[2].Name).To(Equal("Electronic"))
|
||||
})
|
||||
It("does not break on spaces", func() {
|
||||
_, gs := mapper.mapGenres([]string{"New Wave"})
|
||||
Expect(gs).To(HaveLen(1))
|
||||
Expect(gs[0].Name).To(Equal("New Wave"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("mapDates", func() {
|
||||
var md metadata.Tags
|
||||
BeforeEach(func() {
|
||||
mapper = NewMediaFileMapper("/", nil)
|
||||
})
|
||||
Context("when all date fields are provided", func() {
|
||||
BeforeEach(func() {
|
||||
md = metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{
|
||||
"date": []string{"2023-03-01"},
|
||||
"originaldate": []string{"2022-05-10"},
|
||||
"releasedate": []string{"2023-01-15"},
|
||||
})
|
||||
})
|
||||
|
||||
It("should map all date fields correctly", func() {
|
||||
year, date, originalYear, originalDate, releaseYear, releaseDate := mapper.mapDates(md)
|
||||
Expect(year).To(Equal(2023))
|
||||
Expect(date).To(Equal("2023-03-01"))
|
||||
Expect(originalYear).To(Equal(2022))
|
||||
Expect(originalDate).To(Equal("2022-05-10"))
|
||||
Expect(releaseYear).To(Equal(2023))
|
||||
Expect(releaseDate).To(Equal("2023-01-15"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when date field is missing", func() {
|
||||
BeforeEach(func() {
|
||||
md = metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{
|
||||
"originaldate": []string{"2022-05-10"},
|
||||
"releasedate": []string{"2023-01-15"},
|
||||
})
|
||||
})
|
||||
|
||||
It("should fallback to original date if date is missing", func() {
|
||||
year, date, _, _, _, _ := mapper.mapDates(md)
|
||||
Expect(year).To(Equal(2022))
|
||||
Expect(date).To(Equal("2022-05-10"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when original and release dates are missing", func() {
|
||||
BeforeEach(func() {
|
||||
md = metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{
|
||||
"date": []string{"2023-03-01"},
|
||||
})
|
||||
})
|
||||
|
||||
It("should only map the date field", func() {
|
||||
year, date, originalYear, originalDate, releaseYear, releaseDate := mapper.mapDates(md)
|
||||
Expect(year).To(Equal(2023))
|
||||
Expect(date).To(Equal("2023-03-01"))
|
||||
Expect(originalYear).To(BeZero())
|
||||
Expect(originalDate).To(BeEmpty())
|
||||
Expect(releaseYear).To(BeZero())
|
||||
Expect(releaseDate).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when date fields are in an incorrect format", func() {
|
||||
BeforeEach(func() {
|
||||
md = metadata.NewTag("/music/artist/album01/Song.mp3", nil, metadata.ParsedTags{
|
||||
"date": []string{"invalid-date"},
|
||||
})
|
||||
})
|
||||
|
||||
It("should handle invalid date formats gracefully", func() {
|
||||
year, date, _, _, _, _ := mapper.mapDates(md)
|
||||
Expect(year).To(BeZero())
|
||||
Expect(date).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when all date fields are missing", func() {
|
||||
It("should return zero values for all date fields", func() {
|
||||
year, date, originalYear, originalDate, releaseYear, releaseDate := mapper.mapDates(md)
|
||||
Expect(year).To(BeZero())
|
||||
Expect(date).To(BeEmpty())
|
||||
Expect(originalYear).To(BeZero())
|
||||
Expect(originalDate).To(BeEmpty())
|
||||
Expect(releaseYear).To(BeZero())
|
||||
Expect(releaseDate).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,210 +0,0 @@
|
|||
package metadata_test
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
_ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg"
|
||||
_ "github.com/navidrome/navidrome/scanner/metadata/taglib"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Tags", func() {
|
||||
var zero int64 = 0
|
||||
var secondTs int64 = 2500
|
||||
|
||||
makeLyrics := func(synced bool, lang, secondLine string) model.Lyrics {
|
||||
lines := []model.Line{
|
||||
{Value: "This is"},
|
||||
{Value: secondLine},
|
||||
}
|
||||
|
||||
if synced {
|
||||
lines[0].Start = &zero
|
||||
lines[1].Start = &secondTs
|
||||
}
|
||||
|
||||
lyrics := model.Lyrics{
|
||||
Lang: lang,
|
||||
Line: lines,
|
||||
Synced: synced,
|
||||
}
|
||||
|
||||
return lyrics
|
||||
}
|
||||
|
||||
sortLyrics := func(lines model.LyricList) model.LyricList {
|
||||
slices.SortFunc(lines, func(a, b model.Lyrics) int {
|
||||
langDiff := cmp.Compare(a.Lang, b.Lang)
|
||||
if langDiff != 0 {
|
||||
return langDiff
|
||||
}
|
||||
return cmp.Compare(a.Line[1].Value, b.Line[1].Value)
|
||||
})
|
||||
|
||||
return lines
|
||||
}
|
||||
|
||||
compareLyrics := func(m metadata.Tags, expected model.LyricList) {
|
||||
lyrics := model.LyricList{}
|
||||
Expect(json.Unmarshal([]byte(m.Lyrics()), &lyrics)).To(BeNil())
|
||||
Expect(sortLyrics(lyrics)).To(Equal(sortLyrics(expected)))
|
||||
}
|
||||
|
||||
Context("Extract", func() {
|
||||
BeforeEach(func() {
|
||||
conf.Server.Scanner.Extractor = "taglib"
|
||||
})
|
||||
|
||||
It("correctly parses metadata from all files in folder", func() {
|
||||
mds, err := metadata.Extract("tests/fixtures/test.mp3", "tests/fixtures/test.ogg", "tests/fixtures/test.wma")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(3))
|
||||
|
||||
m := mds["tests/fixtures/test.mp3"]
|
||||
Expect(m.Title()).To(Equal("Song"))
|
||||
Expect(m.Album()).To(Equal("Album"))
|
||||
Expect(m.Artist()).To(Equal("Artist"))
|
||||
Expect(m.AlbumArtist()).To(Equal("Album Artist"))
|
||||
Expect(m.Compilation()).To(BeTrue())
|
||||
Expect(m.Genres()).To(Equal([]string{"Rock"}))
|
||||
y, d := m.Date()
|
||||
Expect(y).To(Equal(2014))
|
||||
Expect(d).To(Equal("2014-05-21"))
|
||||
y, d = m.OriginalDate()
|
||||
Expect(y).To(Equal(1996))
|
||||
Expect(d).To(Equal("1996-11-21"))
|
||||
y, d = m.ReleaseDate()
|
||||
Expect(y).To(Equal(2020))
|
||||
Expect(d).To(Equal("2020-12-31"))
|
||||
n, t := m.TrackNumber()
|
||||
Expect(n).To(Equal(2))
|
||||
Expect(t).To(Equal(10))
|
||||
n, t = m.DiscNumber()
|
||||
Expect(n).To(Equal(1))
|
||||
Expect(t).To(Equal(2))
|
||||
Expect(m.HasPicture()).To(BeTrue())
|
||||
Expect(m.Duration()).To(BeNumerically("~", 1.02, 0.01))
|
||||
Expect(m.BitRate()).To(Equal(192))
|
||||
Expect(m.Channels()).To(Equal(2))
|
||||
Expect(m.SampleRate()).To(Equal(44100))
|
||||
Expect(m.FilePath()).To(Equal("tests/fixtures/test.mp3"))
|
||||
Expect(m.Suffix()).To(Equal("mp3"))
|
||||
Expect(m.Size()).To(Equal(int64(51876)))
|
||||
Expect(m.RGAlbumGain()).To(Equal(3.21518))
|
||||
Expect(m.RGAlbumPeak()).To(Equal(0.9125))
|
||||
Expect(m.RGTrackGain()).To(Equal(-1.48))
|
||||
Expect(m.RGTrackPeak()).To(Equal(0.4512))
|
||||
|
||||
m = mds["tests/fixtures/test.ogg"]
|
||||
Expect(err).To(BeNil())
|
||||
Expect(m.Title()).To(Equal("Title"))
|
||||
Expect(m.HasPicture()).To(BeFalse())
|
||||
Expect(m.Duration()).To(BeNumerically("~", 1.04, 0.01))
|
||||
Expect(m.Suffix()).To(Equal("ogg"))
|
||||
Expect(m.FilePath()).To(Equal("tests/fixtures/test.ogg"))
|
||||
Expect(m.Size()).To(Equal(int64(5534)))
|
||||
// TabLib 1.12 returns 18, previous versions return 39.
|
||||
// See https://github.com/taglib/taglib/commit/2f238921824741b2cfe6fbfbfc9701d9827ab06b
|
||||
Expect(m.BitRate()).To(BeElementOf(18, 39, 40, 43, 49))
|
||||
Expect(m.SampleRate()).To(Equal(8000))
|
||||
|
||||
m = mds["tests/fixtures/test.wma"]
|
||||
Expect(err).To(BeNil())
|
||||
Expect(m.Compilation()).To(BeTrue())
|
||||
Expect(m.Title()).To(Equal("Title"))
|
||||
Expect(m.HasPicture()).To(BeFalse())
|
||||
Expect(m.Duration()).To(BeNumerically("~", 1.02, 0.01))
|
||||
Expect(m.Suffix()).To(Equal("wma"))
|
||||
Expect(m.FilePath()).To(Equal("tests/fixtures/test.wma"))
|
||||
Expect(m.Size()).To(Equal(int64(21581)))
|
||||
Expect(m.BitRate()).To(BeElementOf(128))
|
||||
Expect(m.SampleRate()).To(Equal(44100))
|
||||
})
|
||||
|
||||
DescribeTable("Lyrics test",
|
||||
func(file string, langEncoded bool) {
|
||||
path := "tests/fixtures/" + file
|
||||
mds, err := metadata.Extract(path)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(1))
|
||||
|
||||
m := mds[path]
|
||||
lyrics := model.LyricList{
|
||||
makeLyrics(true, "xxx", "English"),
|
||||
makeLyrics(true, "xxx", "unspecified"),
|
||||
}
|
||||
if langEncoded {
|
||||
lyrics[0].Lang = "eng"
|
||||
}
|
||||
compareLyrics(m, lyrics)
|
||||
},
|
||||
|
||||
Entry("Parses AIFF file", "test.aiff", true),
|
||||
Entry("Parses FLAC files", "test.flac", false),
|
||||
Entry("Parses M4A files", "01 Invisible (RED) Edit Version.m4a", false),
|
||||
Entry("Parses OGG Vorbis files", "test.ogg", false),
|
||||
Entry("Parses WAV files", "test.wav", true),
|
||||
Entry("Parses WMA files", "test.wma", false),
|
||||
Entry("Parses WV files", "test.wv", false),
|
||||
)
|
||||
|
||||
It("Should parse mp3 with USLT and SYLT", func() {
|
||||
path := "tests/fixtures/test.mp3"
|
||||
mds, err := metadata.Extract(path)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(1))
|
||||
|
||||
m := mds[path]
|
||||
compareLyrics(m, model.LyricList{
|
||||
makeLyrics(true, "eng", "English SYLT"),
|
||||
makeLyrics(true, "eng", "English"),
|
||||
makeLyrics(true, "xxx", "unspecified SYLT"),
|
||||
makeLyrics(true, "xxx", "unspecified"),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// Only run these tests if FFmpeg is available
|
||||
FFmpegContext := XContext
|
||||
if ffmpeg.New().IsAvailable() {
|
||||
FFmpegContext = Context
|
||||
}
|
||||
FFmpegContext("Extract with FFmpeg", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.Scanner.Extractor = "ffmpeg"
|
||||
})
|
||||
|
||||
DescribeTable("Lyrics test",
|
||||
func(file string) {
|
||||
path := "tests/fixtures/" + file
|
||||
mds, err := metadata.Extract(path)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(1))
|
||||
|
||||
m := mds[path]
|
||||
compareLyrics(m, model.LyricList{
|
||||
makeLyrics(true, "eng", "English"),
|
||||
makeLyrics(true, "xxx", "unspecified"),
|
||||
})
|
||||
},
|
||||
|
||||
Entry("Parses AIFF file", "test.aiff"),
|
||||
Entry("Parses MP3 files", "test.mp3"),
|
||||
// Disabled, because it fails in pipeline
|
||||
// Entry("Parses WAV files", "test.wav"),
|
||||
|
||||
// FFMPEG behaves very weirdly for multivalued tags for non-ID3
|
||||
// Specifically, they are separated by ";, which is indistinguishable
|
||||
// from other fields
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
//go:build !windows
|
||||
|
||||
package taglib
|
||||
|
||||
import "C"
|
||||
|
||||
func getFilename(s string) *C.char {
|
||||
return C.CString(s)
|
||||
}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
//go:build windows
|
||||
|
||||
package taglib
|
||||
|
||||
// From https://github.com/orofarne/gowchar
|
||||
|
||||
/*
|
||||
#include <wchar.h>
|
||||
|
||||
const size_t SIZEOF_WCHAR_T = sizeof(wchar_t);
|
||||
|
||||
void gowchar_set (wchar_t *arr, int pos, wchar_t val)
|
||||
{
|
||||
arr[pos] = val;
|
||||
}
|
||||
|
||||
wchar_t gowchar_get (wchar_t *arr, int pos)
|
||||
{
|
||||
return arr[pos];
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var SIZEOF_WCHAR_T C.size_t = C.size_t(C.SIZEOF_WCHAR_T)
|
||||
|
||||
func getFilename(s string) *C.wchar_t {
|
||||
wstr, _ := StringToWcharT(s)
|
||||
return wstr
|
||||
}
|
||||
|
||||
func StringToWcharT(s string) (*C.wchar_t, C.size_t) {
|
||||
switch SIZEOF_WCHAR_T {
|
||||
case 2:
|
||||
return stringToWchar2(s) // Windows
|
||||
case 4:
|
||||
return stringToWchar4(s) // Unix
|
||||
default:
|
||||
panic(fmt.Sprintf("Invalid sizeof(wchar_t) = %v", SIZEOF_WCHAR_T))
|
||||
}
|
||||
panic("?!!")
|
||||
}
|
||||
|
||||
// Windows
|
||||
func stringToWchar2(s string) (*C.wchar_t, C.size_t) {
|
||||
var slen int
|
||||
s1 := s
|
||||
for len(s1) > 0 {
|
||||
r, size := utf8.DecodeRuneInString(s1)
|
||||
if er, _ := utf16.EncodeRune(r); er == '\uFFFD' {
|
||||
slen += 1
|
||||
} else {
|
||||
slen += 2
|
||||
}
|
||||
s1 = s1[size:]
|
||||
}
|
||||
slen++ // \0
|
||||
res := C.malloc(C.size_t(slen) * SIZEOF_WCHAR_T)
|
||||
var i int
|
||||
for len(s) > 0 {
|
||||
r, size := utf8.DecodeRuneInString(s)
|
||||
if r1, r2 := utf16.EncodeRune(r); r1 != '\uFFFD' {
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(i), C.wchar_t(r1))
|
||||
i++
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(i), C.wchar_t(r2))
|
||||
i++
|
||||
} else {
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(i), C.wchar_t(r))
|
||||
i++
|
||||
}
|
||||
s = s[size:]
|
||||
}
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(slen-1), C.wchar_t(0)) // \0
|
||||
return (*C.wchar_t)(res), C.size_t(slen)
|
||||
}
|
||||
|
||||
// Unix
|
||||
func stringToWchar4(s string) (*C.wchar_t, C.size_t) {
|
||||
slen := utf8.RuneCountInString(s)
|
||||
slen++ // \0
|
||||
res := C.malloc(C.size_t(slen) * SIZEOF_WCHAR_T)
|
||||
var i int
|
||||
for len(s) > 0 {
|
||||
r, size := utf8.DecodeRuneInString(s)
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(i), C.wchar_t(r))
|
||||
s = s[size:]
|
||||
i++
|
||||
}
|
||||
C.gowchar_set((*C.wchar_t)(res), C.int(slen-1), C.wchar_t(0)) // \0
|
||||
return (*C.wchar_t)(res), C.size_t(slen)
|
||||
}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
package taglib
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
)
|
||||
|
||||
const ExtractorID = "taglib"
|
||||
|
||||
type Extractor struct{}
|
||||
|
||||
func (e *Extractor) Parse(paths ...string) (map[string]metadata.ParsedTags, error) {
|
||||
fileTags := map[string]metadata.ParsedTags{}
|
||||
for _, path := range paths {
|
||||
tags, err := e.extractMetadata(path)
|
||||
if !errors.Is(err, os.ErrPermission) {
|
||||
fileTags[path] = tags
|
||||
}
|
||||
}
|
||||
return fileTags, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) CustomMappings() metadata.ParsedTags {
|
||||
return metadata.ParsedTags{
|
||||
"title": {"titlesort"},
|
||||
"album": {"albumsort"},
|
||||
"artist": {"artistsort"},
|
||||
"tracknumber": {"trck", "_track"},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Version() string {
|
||||
return Version()
|
||||
}
|
||||
|
||||
func (e *Extractor) extractMetadata(filePath string) (metadata.ParsedTags, error) {
|
||||
tags, err := Read(filePath)
|
||||
if err != nil {
|
||||
log.Warn("TagLib: Error reading metadata from file. Skipping", "filePath", filePath, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if length, ok := tags["lengthinmilliseconds"]; ok && len(length) > 0 {
|
||||
millis, _ := strconv.Atoi(length[0])
|
||||
if duration := float64(millis) / 1000.0; duration > 0 {
|
||||
tags["duration"] = []string{strconv.FormatFloat(duration, 'f', 2, 32)}
|
||||
}
|
||||
}
|
||||
// Adjust some ID3 tags
|
||||
parseTIPL(tags)
|
||||
delete(tags, "tmcl") // TMCL is already parsed by TagLib
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// These are the only roles we support, based on Picard's tag map:
|
||||
// https://picard-docs.musicbrainz.org/downloads/MusicBrainz_Picard_Tag_Map.html
|
||||
var tiplMapping = map[string]string{
|
||||
"arranger": "arranger",
|
||||
"engineer": "engineer",
|
||||
"producer": "producer",
|
||||
"mix": "mixer",
|
||||
"dj-mix": "djmixer",
|
||||
}
|
||||
|
||||
// parseTIPL parses the ID3v2.4 TIPL frame string, which is received from TagLib in the format
|
||||
//
|
||||
// "arranger Andrew Powell engineer Chris Blair engineer Pat Stapley producer Eric Woolfson".
|
||||
//
|
||||
// and breaks it down into a map of roles and names, e.g.:
|
||||
//
|
||||
// {"arranger": ["Andrew Powell"], "engineer": ["Chris Blair", "Pat Stapley"], "producer": ["Eric Woolfson"]}.
|
||||
func parseTIPL(tags metadata.ParsedTags) {
|
||||
tipl := tags["tipl"]
|
||||
if len(tipl) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
addRole := func(tags metadata.ParsedTags, currentRole string, currentValue []string) {
|
||||
if currentRole != "" && len(currentValue) > 0 {
|
||||
role := tiplMapping[currentRole]
|
||||
tags[role] = append(tags[currentRole], strings.Join(currentValue, " "))
|
||||
}
|
||||
}
|
||||
|
||||
var currentRole string
|
||||
var currentValue []string
|
||||
for _, part := range strings.Split(tipl[0], " ") {
|
||||
if _, ok := tiplMapping[part]; ok {
|
||||
addRole(tags, currentRole, currentValue)
|
||||
currentRole = part
|
||||
currentValue = nil
|
||||
continue
|
||||
}
|
||||
currentValue = append(currentValue, part)
|
||||
}
|
||||
addRole(tags, currentRole, currentValue)
|
||||
delete(tags, "tipl")
|
||||
}
|
||||
|
||||
func init() {
|
||||
metadata.RegisterExtractor(ExtractorID, &Extractor{})
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
package taglib
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestTagLib(t *testing.T) {
|
||||
tests.Init(t, true)
|
||||
log.SetLevel(log.LevelFatal)
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "TagLib Suite")
|
||||
}
|
||||
|
|
@ -1,280 +0,0 @@
|
|||
package taglib
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
"github.com/navidrome/navidrome/utils"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Extractor", func() {
|
||||
var e *Extractor
|
||||
|
||||
BeforeEach(func() {
|
||||
e = &Extractor{}
|
||||
})
|
||||
|
||||
Describe("Parse", func() {
|
||||
It("correctly parses metadata from all files in folder", func() {
|
||||
mds, err := e.Parse(
|
||||
"tests/fixtures/test.mp3",
|
||||
"tests/fixtures/test.ogg",
|
||||
)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(2))
|
||||
|
||||
// Test MP3
|
||||
m := mds["tests/fixtures/test.mp3"]
|
||||
Expect(m).To(HaveKeyWithValue("title", []string{"Song", "Song"}))
|
||||
Expect(m).To(HaveKeyWithValue("album", []string{"Album", "Album"}))
|
||||
Expect(m).To(HaveKeyWithValue("artist", []string{"Artist", "Artist"}))
|
||||
Expect(m).To(HaveKeyWithValue("albumartist", []string{"Album Artist"}))
|
||||
|
||||
Expect(m).To(Or(
|
||||
HaveKeyWithValue("compilation", []string{"1"}),
|
||||
HaveKeyWithValue("tcmp", []string{"1"}))) // Compilation
|
||||
Expect(m).To(HaveKeyWithValue("genre", []string{"Rock"}))
|
||||
Expect(m).To(HaveKeyWithValue("date", []string{"2014-05-21", "2014"}))
|
||||
Expect(m).To(HaveKeyWithValue("originaldate", []string{"1996-11-21"}))
|
||||
Expect(m).To(HaveKeyWithValue("releasedate", []string{"2020-12-31"}))
|
||||
Expect(m).To(HaveKeyWithValue("discnumber", []string{"1/2"}))
|
||||
Expect(m).To(HaveKeyWithValue("has_picture", []string{"true"}))
|
||||
Expect(m).To(HaveKeyWithValue("duration", []string{"1.02"}))
|
||||
Expect(m).To(HaveKeyWithValue("bitrate", []string{"192"}))
|
||||
Expect(m).To(HaveKeyWithValue("channels", []string{"2"}))
|
||||
Expect(m).To(HaveKeyWithValue("samplerate", []string{"44100"}))
|
||||
Expect(m).To(HaveKeyWithValue("comment", []string{"Comment1\nComment2"}))
|
||||
Expect(m).ToNot(HaveKey("lyrics"))
|
||||
Expect(m).To(Or(HaveKeyWithValue("lyrics-eng", []string{
|
||||
"[00:00.00]This is\n[00:02.50]English SYLT\n",
|
||||
"[00:00.00]This is\n[00:02.50]English",
|
||||
}), HaveKeyWithValue("lyrics-eng", []string{
|
||||
"[00:00.00]This is\n[00:02.50]English",
|
||||
"[00:00.00]This is\n[00:02.50]English SYLT\n",
|
||||
})))
|
||||
Expect(m).To(Or(HaveKeyWithValue("lyrics-xxx", []string{
|
||||
"[00:00.00]This is\n[00:02.50]unspecified SYLT\n",
|
||||
"[00:00.00]This is\n[00:02.50]unspecified",
|
||||
}), HaveKeyWithValue("lyrics-xxx", []string{
|
||||
"[00:00.00]This is\n[00:02.50]unspecified",
|
||||
"[00:00.00]This is\n[00:02.50]unspecified SYLT\n",
|
||||
})))
|
||||
Expect(m).To(HaveKeyWithValue("bpm", []string{"123"}))
|
||||
Expect(m).To(HaveKeyWithValue("replaygain_album_gain", []string{"+3.21518 dB"}))
|
||||
Expect(m).To(HaveKeyWithValue("replaygain_album_peak", []string{"0.9125"}))
|
||||
Expect(m).To(HaveKeyWithValue("replaygain_track_gain", []string{"-1.48 dB"}))
|
||||
Expect(m).To(HaveKeyWithValue("replaygain_track_peak", []string{"0.4512"}))
|
||||
|
||||
Expect(m).To(HaveKeyWithValue("tracknumber", []string{"2/10"}))
|
||||
m = m.Map(e.CustomMappings())
|
||||
Expect(m).To(HaveKeyWithValue("tracknumber", []string{"2/10", "2/10", "2"}))
|
||||
|
||||
// Test OGG
|
||||
m = mds["tests/fixtures/test.ogg"]
|
||||
Expect(err).To(BeNil())
|
||||
Expect(m).ToNot(HaveKey("has_picture"))
|
||||
Expect(m).To(HaveKeyWithValue("duration", []string{"1.04"}))
|
||||
Expect(m).To(HaveKeyWithValue("fbpm", []string{"141.7"}))
|
||||
Expect(m).To(HaveKeyWithValue("samplerate", []string{"8000"}))
|
||||
|
||||
// TabLib 1.12 returns 18, previous versions return 39.
|
||||
// See https://github.com/taglib/taglib/commit/2f238921824741b2cfe6fbfbfc9701d9827ab06b
|
||||
Expect(m).To(HaveKey("bitrate"))
|
||||
Expect(m["bitrate"][0]).To(BeElementOf("18", "39", "40", "43", "49"))
|
||||
})
|
||||
|
||||
DescribeTable("Format-Specific tests",
|
||||
func(file, duration, channels, samplerate, albumGain, albumPeak, trackGain, trackPeak string, id3Lyrics bool) {
|
||||
file = "tests/fixtures/" + file
|
||||
mds, err := e.Parse(file)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(1))
|
||||
|
||||
m := mds[file]
|
||||
|
||||
Expect(m["replaygain_album_gain"]).To(ContainElement(albumGain))
|
||||
Expect(m["replaygain_album_peak"]).To(ContainElement(albumPeak))
|
||||
Expect(m["replaygain_track_gain"]).To(ContainElement(trackGain))
|
||||
Expect(m["replaygain_track_peak"]).To(ContainElement(trackPeak))
|
||||
|
||||
Expect(m).To(HaveKeyWithValue("title", []string{"Title", "Title"}))
|
||||
Expect(m).To(HaveKeyWithValue("album", []string{"Album", "Album"}))
|
||||
Expect(m).To(HaveKeyWithValue("artist", []string{"Artist", "Artist"}))
|
||||
Expect(m).To(HaveKeyWithValue("albumartist", []string{"Album Artist"}))
|
||||
Expect(m).To(HaveKeyWithValue("genre", []string{"Rock"}))
|
||||
Expect(m).To(HaveKeyWithValue("date", []string{"2014", "2014"}))
|
||||
|
||||
// Special for M4A, do not catch keys that have no actual name
|
||||
Expect(m).ToNot(HaveKey(""))
|
||||
|
||||
Expect(m).To(HaveKey("discnumber"))
|
||||
discno := m["discnumber"]
|
||||
Expect(discno).To(HaveLen(1))
|
||||
Expect(discno[0]).To(BeElementOf([]string{"1", "1/2"}))
|
||||
|
||||
// WMA does not have a "compilation" tag, but "wm/iscompilation"
|
||||
if _, ok := m["compilation"]; ok {
|
||||
Expect(m).To(HaveKeyWithValue("compilation", []string{"1"}))
|
||||
} else {
|
||||
Expect(m).To(HaveKeyWithValue("wm/iscompilation", []string{"1"}))
|
||||
}
|
||||
|
||||
Expect(m).NotTo(HaveKeyWithValue("has_picture", []string{"true"}))
|
||||
Expect(m).To(HaveKeyWithValue("duration", []string{duration}))
|
||||
|
||||
Expect(m).To(HaveKeyWithValue("channels", []string{channels}))
|
||||
Expect(m).To(HaveKeyWithValue("samplerate", []string{samplerate}))
|
||||
Expect(m).To(HaveKeyWithValue("comment", []string{"Comment1\nComment2"}))
|
||||
|
||||
if id3Lyrics {
|
||||
Expect(m).To(HaveKeyWithValue("lyrics-eng", []string{
|
||||
"[00:00.00]This is\n[00:02.50]English",
|
||||
}))
|
||||
Expect(m).To(HaveKeyWithValue("lyrics-xxx", []string{
|
||||
"[00:00.00]This is\n[00:02.50]unspecified",
|
||||
}))
|
||||
} else {
|
||||
Expect(m).To(HaveKeyWithValue("lyrics", []string{
|
||||
"[00:00.00]This is\n[00:02.50]unspecified",
|
||||
"[00:00.00]This is\n[00:02.50]English",
|
||||
}))
|
||||
}
|
||||
|
||||
Expect(m).To(HaveKeyWithValue("bpm", []string{"123"}))
|
||||
|
||||
Expect(m).To(HaveKey("tracknumber"))
|
||||
trackNo := m["tracknumber"]
|
||||
Expect(trackNo).To(HaveLen(1))
|
||||
Expect(trackNo[0]).To(BeElementOf([]string{"3", "3/10"}))
|
||||
},
|
||||
|
||||
// ffmpeg -f lavfi -i "sine=frequency=1200:duration=1" test.flac
|
||||
Entry("correctly parses flac tags", "test.flac", "1.00", "1", "44100", "+4.06 dB", "0.12496948", "+4.06 dB", "0.12496948", false),
|
||||
|
||||
Entry("Correctly parses m4a (aac) gain tags", "01 Invisible (RED) Edit Version.m4a", "1.04", "2", "44100", "0.37", "0.48", "0.37", "0.48", false),
|
||||
Entry("Correctly parses m4a (aac) gain tags (uppercase)", "test.m4a", "1.04", "2", "44100", "0.37", "0.48", "0.37", "0.48", false),
|
||||
Entry("correctly parses ogg (vorbis) tags", "test.ogg", "1.04", "2", "8000", "+7.64 dB", "0.11772506", "+7.64 dB", "0.11772506", false),
|
||||
|
||||
// ffmpeg -f lavfi -i "sine=frequency=900:duration=1" test.wma
|
||||
// Weird note: for the tag parsing to work, the lyrics are actually stored in the reverse order
|
||||
Entry("correctly parses wma/asf tags", "test.wma", "1.02", "1", "44100", "3.27 dB", "0.132914", "3.27 dB", "0.132914", false),
|
||||
|
||||
// ffmpeg -f lavfi -i "sine=frequency=800:duration=1" test.wv
|
||||
Entry("correctly parses wv (wavpak) tags", "test.wv", "1.00", "1", "44100", "3.43 dB", "0.125061", "3.43 dB", "0.125061", false),
|
||||
|
||||
// ffmpeg -f lavfi -i "sine=frequency=1000:duration=1" test.wav
|
||||
Entry("correctly parses wav tags", "test.wav", "1.00", "1", "44100", "3.06 dB", "0.125056", "3.06 dB", "0.125056", true),
|
||||
|
||||
// ffmpeg -f lavfi -i "sine=frequency=1400:duration=1" test.aiff
|
||||
Entry("correctly parses aiff tags", "test.aiff", "1.00", "1", "44100", "2.00 dB", "0.124972", "2.00 dB", "0.124972", true),
|
||||
)
|
||||
|
||||
// Skip these tests when running as root
|
||||
Context("Access Forbidden", func() {
|
||||
var accessForbiddenFile string
|
||||
var RegularUserContext = XContext
|
||||
var isRegularUser = os.Getuid() != 0
|
||||
if isRegularUser {
|
||||
RegularUserContext = Context
|
||||
}
|
||||
|
||||
// Only run permission tests if we are not root
|
||||
RegularUserContext("when run without root privileges", func() {
|
||||
BeforeEach(func() {
|
||||
accessForbiddenFile = utils.TempFileName("access_forbidden-", ".mp3")
|
||||
|
||||
f, err := os.OpenFile(accessForbiddenFile, os.O_WRONLY|os.O_CREATE, 0222)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
DeferCleanup(func() {
|
||||
Expect(f.Close()).To(Succeed())
|
||||
Expect(os.Remove(accessForbiddenFile)).To(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
It("correctly handle unreadable file due to insufficient read permission", func() {
|
||||
_, err := e.extractMetadata(accessForbiddenFile)
|
||||
Expect(err).To(MatchError(os.ErrPermission))
|
||||
})
|
||||
|
||||
It("skips the file if it cannot be read", func() {
|
||||
files := []string{
|
||||
"tests/fixtures/test.mp3",
|
||||
"tests/fixtures/test.ogg",
|
||||
accessForbiddenFile,
|
||||
}
|
||||
mds, err := e.Parse(files...)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(2))
|
||||
Expect(mds).ToNot(HaveKey(accessForbiddenFile))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Describe("Error Checking", func() {
|
||||
It("returns a generic ErrPath if file does not exist", func() {
|
||||
testFilePath := "tests/fixtures/NON_EXISTENT.ogg"
|
||||
_, err := e.extractMetadata(testFilePath)
|
||||
Expect(err).To(MatchError(fs.ErrNotExist))
|
||||
})
|
||||
It("does not throw a SIGSEGV error when reading a file with an invalid frame", func() {
|
||||
// File has an empty TDAT frame
|
||||
md, err := e.extractMetadata("tests/fixtures/invalid-files/test-invalid-frame.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(md).To(HaveKeyWithValue("albumartist", []string{"Elvis Presley"}))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("parseTIPL", func() {
|
||||
var tags metadata.ParsedTags
|
||||
|
||||
BeforeEach(func() {
|
||||
tags = metadata.ParsedTags{}
|
||||
})
|
||||
|
||||
Context("when the TIPL string is populated", func() {
|
||||
It("correctly parses roles and names", func() {
|
||||
tags["tipl"] = []string{"arranger Andrew Powell dj-mix François Kevorkian engineer Chris Blair"}
|
||||
parseTIPL(tags)
|
||||
Expect(tags["arranger"]).To(ConsistOf("Andrew Powell"))
|
||||
Expect(tags["engineer"]).To(ConsistOf("Chris Blair"))
|
||||
Expect(tags["djmixer"]).To(ConsistOf("François Kevorkian"))
|
||||
})
|
||||
|
||||
It("handles multiple names for a single role", func() {
|
||||
tags["tipl"] = []string{"engineer Pat Stapley producer Eric Woolfson engineer Chris Blair"}
|
||||
parseTIPL(tags)
|
||||
Expect(tags["producer"]).To(ConsistOf("Eric Woolfson"))
|
||||
Expect(tags["engineer"]).To(ConsistOf("Pat Stapley", "Chris Blair"))
|
||||
})
|
||||
|
||||
It("discards roles without names", func() {
|
||||
tags["tipl"] = []string{"engineer Pat Stapley producer engineer Chris Blair"}
|
||||
parseTIPL(tags)
|
||||
Expect(tags).ToNot(HaveKey("producer"))
|
||||
Expect(tags["engineer"]).To(ConsistOf("Pat Stapley", "Chris Blair"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when the TIPL string is empty", func() {
|
||||
It("does nothing", func() {
|
||||
tags["tipl"] = []string{""}
|
||||
parseTIPL(tags)
|
||||
Expect(tags).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when the TIPL is not present", func() {
|
||||
It("does nothing", func() {
|
||||
parseTIPL(tags)
|
||||
Expect(tags).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
|
@ -1,240 +0,0 @@
|
|||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <typeinfo>
|
||||
|
||||
#define TAGLIB_STATIC
|
||||
#include <aifffile.h>
|
||||
#include <asffile.h>
|
||||
#include <fileref.h>
|
||||
#include <flacfile.h>
|
||||
#include <id3v2tag.h>
|
||||
#include <unsynchronizedlyricsframe.h>
|
||||
#include <synchronizedlyricsframe.h>
|
||||
#include <mp4file.h>
|
||||
#include <mpegfile.h>
|
||||
#include <opusfile.h>
|
||||
#include <tpropertymap.h>
|
||||
#include <vorbisfile.h>
|
||||
#include <wavfile.h>
|
||||
|
||||
#include "taglib_wrapper.h"
|
||||
|
||||
char has_cover(const TagLib::FileRef f);
|
||||
|
||||
static char TAGLIB_VERSION[16];
|
||||
|
||||
char* taglib_version() {
|
||||
snprintf((char *)TAGLIB_VERSION, 16, "%d.%d.%d", TAGLIB_MAJOR_VERSION, TAGLIB_MINOR_VERSION, TAGLIB_PATCH_VERSION);
|
||||
return (char *)TAGLIB_VERSION;
|
||||
}
|
||||
|
||||
int taglib_read(const FILENAME_CHAR_T *filename, unsigned long id) {
|
||||
TagLib::FileRef f(filename, true, TagLib::AudioProperties::Fast);
|
||||
|
||||
if (f.isNull()) {
|
||||
return TAGLIB_ERR_PARSE;
|
||||
}
|
||||
|
||||
if (!f.audioProperties()) {
|
||||
return TAGLIB_ERR_AUDIO_PROPS;
|
||||
}
|
||||
|
||||
// Add audio properties to the tags
|
||||
const TagLib::AudioProperties *props(f.audioProperties());
|
||||
go_map_put_int(id, (char *)"duration", props->lengthInSeconds());
|
||||
go_map_put_int(id, (char *)"lengthinmilliseconds", props->lengthInMilliseconds());
|
||||
go_map_put_int(id, (char *)"bitrate", props->bitrate());
|
||||
go_map_put_int(id, (char *)"channels", props->channels());
|
||||
go_map_put_int(id, (char *)"samplerate", props->sampleRate());
|
||||
|
||||
// Create a map to collect all the tags
|
||||
TagLib::PropertyMap tags = f.file()->properties();
|
||||
|
||||
// Make sure at least the basic properties are extracted
|
||||
TagLib::Tag *basic = f.file()->tag();
|
||||
if (!basic->isEmpty()) {
|
||||
if (!basic->title().isEmpty()) {
|
||||
tags.insert("title", basic->title());
|
||||
}
|
||||
if (!basic->artist().isEmpty()) {
|
||||
tags.insert("artist", basic->artist());
|
||||
}
|
||||
if (!basic->album().isEmpty()) {
|
||||
tags.insert("album", basic->album());
|
||||
}
|
||||
if (basic->year() > 0) {
|
||||
tags.insert("date", TagLib::String::number(basic->year()));
|
||||
}
|
||||
if (basic->track() > 0) {
|
||||
tags.insert("_track", TagLib::String::number(basic->track()));
|
||||
}
|
||||
}
|
||||
|
||||
TagLib::ID3v2::Tag *id3Tags = NULL;
|
||||
|
||||
// Get some extended/non-standard ID3-only tags (ex: iTunes extended frames)
|
||||
TagLib::MPEG::File *mp3File(dynamic_cast<TagLib::MPEG::File *>(f.file()));
|
||||
if (mp3File != NULL) {
|
||||
id3Tags = mp3File->ID3v2Tag();
|
||||
}
|
||||
|
||||
if (id3Tags == NULL) {
|
||||
TagLib::RIFF::WAV::File *wavFile(dynamic_cast<TagLib::RIFF::WAV::File *>(f.file()));
|
||||
if (wavFile != NULL && wavFile->hasID3v2Tag()) {
|
||||
id3Tags = wavFile->ID3v2Tag();
|
||||
}
|
||||
}
|
||||
|
||||
if (id3Tags == NULL) {
|
||||
TagLib::RIFF::AIFF::File *aiffFile(dynamic_cast<TagLib::RIFF::AIFF::File *>(f.file()));
|
||||
if (aiffFile && aiffFile->hasID3v2Tag()) {
|
||||
id3Tags = aiffFile->tag();
|
||||
}
|
||||
}
|
||||
|
||||
// Yes, it is possible to have ID3v2 tags in FLAC. However, that can cause problems
|
||||
// with many players, so they will not be parsed
|
||||
|
||||
if (id3Tags != NULL) {
|
||||
const auto &frames = id3Tags->frameListMap();
|
||||
|
||||
for (const auto &kv: frames) {
|
||||
if (kv.first == "USLT") {
|
||||
for (const auto &tag: kv.second) {
|
||||
TagLib::ID3v2::UnsynchronizedLyricsFrame *frame = dynamic_cast<TagLib::ID3v2::UnsynchronizedLyricsFrame *>(tag);
|
||||
if (frame == NULL) continue;
|
||||
|
||||
tags.erase("LYRICS");
|
||||
|
||||
const auto bv = frame->language();
|
||||
char language[4] = {'x', 'x', 'x', '\0'};
|
||||
if (bv.size() == 3) {
|
||||
strncpy(language, bv.data(), 3);
|
||||
}
|
||||
|
||||
char *val = (char *)frame->text().toCString(true);
|
||||
|
||||
go_map_put_lyrics(id, language, val);
|
||||
}
|
||||
} else if (kv.first == "SYLT") {
|
||||
for (const auto &tag: kv.second) {
|
||||
TagLib::ID3v2::SynchronizedLyricsFrame *frame = dynamic_cast<TagLib::ID3v2::SynchronizedLyricsFrame *>(tag);
|
||||
if (frame == NULL) continue;
|
||||
|
||||
const auto bv = frame->language();
|
||||
char language[4] = {'x', 'x', 'x', '\0'};
|
||||
if (bv.size() == 3) {
|
||||
strncpy(language, bv.data(), 3);
|
||||
}
|
||||
|
||||
const auto format = frame->timestampFormat();
|
||||
if (format == TagLib::ID3v2::SynchronizedLyricsFrame::AbsoluteMilliseconds) {
|
||||
|
||||
for (const auto &line: frame->synchedText()) {
|
||||
char *text = (char *)line.text.toCString(true);
|
||||
go_map_put_lyric_line(id, language, text, line.time);
|
||||
}
|
||||
} else if (format == TagLib::ID3v2::SynchronizedLyricsFrame::AbsoluteMpegFrames) {
|
||||
const int sampleRate = props->sampleRate();
|
||||
|
||||
if (sampleRate != 0) {
|
||||
for (const auto &line: frame->synchedText()) {
|
||||
const int timeInMs = (line.time * 1000) / sampleRate;
|
||||
char *text = (char *)line.text.toCString(true);
|
||||
go_map_put_lyric_line(id, language, text, timeInMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (!kv.second.isEmpty()) {
|
||||
tags.insert(kv.first, kv.second.front()->toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// M4A may have some iTunes specific tags
|
||||
TagLib::MP4::File *m4afile(dynamic_cast<TagLib::MP4::File *>(f.file()));
|
||||
if (m4afile != NULL) {
|
||||
const auto itemListMap = m4afile->tag()->itemMap();
|
||||
for (const auto item: itemListMap) {
|
||||
char *key = (char *)item.first.toCString(true);
|
||||
for (const auto value: item.second.toStringList()) {
|
||||
char *val = (char *)value.toCString(true);
|
||||
go_map_put_m4a_str(id, key, val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WMA/ASF files may have additional tags not captured by the general iterator
|
||||
TagLib::ASF::File *asfFile(dynamic_cast<TagLib::ASF::File *>(f.file()));
|
||||
if (asfFile != NULL) {
|
||||
const TagLib::ASF::Tag *asfTags{asfFile->tag()};
|
||||
const auto itemListMap = asfTags->attributeListMap();
|
||||
for (const auto item : itemListMap) {
|
||||
tags.insert(item.first, item.second.front().toString());
|
||||
}
|
||||
}
|
||||
|
||||
// Send all collected tags to the Go map
|
||||
for (TagLib::PropertyMap::ConstIterator i = tags.begin(); i != tags.end();
|
||||
++i) {
|
||||
char *key = (char *)i->first.toCString(true);
|
||||
for (TagLib::StringList::ConstIterator j = i->second.begin();
|
||||
j != i->second.end(); ++j) {
|
||||
char *val = (char *)(*j).toCString(true);
|
||||
go_map_put_str(id, key, val);
|
||||
}
|
||||
}
|
||||
|
||||
// Cover art has to be handled separately
|
||||
if (has_cover(f)) {
|
||||
go_map_put_str(id, (char *)"has_picture", (char *)"true");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Detect if the file has cover art. Returns 1 if the file has cover art, 0 otherwise.
|
||||
char has_cover(const TagLib::FileRef f) {
|
||||
char hasCover = 0;
|
||||
// ----- MP3
|
||||
if (TagLib::MPEG::File *
|
||||
mp3File{dynamic_cast<TagLib::MPEG::File *>(f.file())}) {
|
||||
if (mp3File->ID3v2Tag()) {
|
||||
const auto &frameListMap{mp3File->ID3v2Tag()->frameListMap()};
|
||||
hasCover = !frameListMap["APIC"].isEmpty();
|
||||
}
|
||||
}
|
||||
// ----- FLAC
|
||||
else if (TagLib::FLAC::File *
|
||||
flacFile{dynamic_cast<TagLib::FLAC::File *>(f.file())}) {
|
||||
hasCover = !flacFile->pictureList().isEmpty();
|
||||
}
|
||||
// ----- MP4
|
||||
else if (TagLib::MP4::File *
|
||||
mp4File{dynamic_cast<TagLib::MP4::File *>(f.file())}) {
|
||||
auto &coverItem{mp4File->tag()->itemMap()["covr"]};
|
||||
TagLib::MP4::CoverArtList coverArtList{coverItem.toCoverArtList()};
|
||||
hasCover = !coverArtList.isEmpty();
|
||||
}
|
||||
// ----- Ogg
|
||||
else if (TagLib::Ogg::Vorbis::File *
|
||||
vorbisFile{dynamic_cast<TagLib::Ogg::Vorbis::File *>(f.file())}) {
|
||||
hasCover = !vorbisFile->tag()->pictureList().isEmpty();
|
||||
}
|
||||
// ----- Opus
|
||||
else if (TagLib::Ogg::Opus::File *
|
||||
opusFile{dynamic_cast<TagLib::Ogg::Opus::File *>(f.file())}) {
|
||||
hasCover = !opusFile->tag()->pictureList().isEmpty();
|
||||
}
|
||||
// ----- WMA
|
||||
if (TagLib::ASF::File *
|
||||
asfFile{dynamic_cast<TagLib::ASF::File *>(f.file())}) {
|
||||
const TagLib::ASF::Tag *tag{asfFile->tag()};
|
||||
hasCover = tag && tag->attributeListMap().contains("WM/Picture");
|
||||
}
|
||||
|
||||
return hasCover;
|
||||
}
|
||||
|
|
@ -1,166 +0,0 @@
|
|||
package taglib
|
||||
|
||||
/*
|
||||
#cgo pkg-config: --define-prefix taglib
|
||||
#cgo illumos LDFLAGS: -lstdc++ -lsendfile
|
||||
#cgo linux darwin CXXFLAGS: -std=c++11
|
||||
#cgo darwin LDFLAGS: -L/opt/homebrew/opt/taglib/lib
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "taglib_wrapper.h"
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
)
|
||||
|
||||
const iTunesKeyPrefix = "----:com.apple.itunes:"
|
||||
|
||||
func Version() string {
|
||||
return C.GoString(C.taglib_version())
|
||||
}
|
||||
|
||||
func Read(filename string) (tags map[string][]string, err error) {
|
||||
// Do not crash on failures in the C code/library
|
||||
debug.SetPanicOnFault(true)
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error("TagLib: recovered from panic when reading tags", "file", filename, "error", r)
|
||||
err = fmt.Errorf("TagLib: recovered from panic: %s", r)
|
||||
}
|
||||
}()
|
||||
|
||||
fp := getFilename(filename)
|
||||
defer C.free(unsafe.Pointer(fp))
|
||||
id, m := newMap()
|
||||
defer deleteMap(id)
|
||||
|
||||
log.Trace("TagLib: reading tags", "filename", filename, "map_id", id)
|
||||
res := C.taglib_read(fp, C.ulong(id))
|
||||
switch res {
|
||||
case C.TAGLIB_ERR_PARSE:
|
||||
// Check additional case whether the file is unreadable due to permission
|
||||
file, fileErr := os.OpenFile(filename, os.O_RDONLY, 0600)
|
||||
defer file.Close()
|
||||
|
||||
if os.IsPermission(fileErr) {
|
||||
return nil, fmt.Errorf("navidrome does not have permission: %w", fileErr)
|
||||
} else if fileErr != nil {
|
||||
return nil, fmt.Errorf("cannot parse file media file: %w", fileErr)
|
||||
} else {
|
||||
return nil, fmt.Errorf("cannot parse file media file")
|
||||
}
|
||||
case C.TAGLIB_ERR_AUDIO_PROPS:
|
||||
return nil, fmt.Errorf("can't get audio properties from file")
|
||||
}
|
||||
if log.IsGreaterOrEqualTo(log.LevelDebug) {
|
||||
j, _ := json.Marshal(m)
|
||||
log.Trace("TagLib: read tags", "tags", string(j), "filename", filename, "id", id)
|
||||
} else {
|
||||
log.Trace("TagLib: read tags", "tags", m, "filename", filename, "id", id)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var lock sync.RWMutex
|
||||
var allMaps = make(map[uint32]map[string][]string)
|
||||
var mapsNextID uint32
|
||||
|
||||
func newMap() (id uint32, m map[string][]string) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
id = mapsNextID
|
||||
mapsNextID++
|
||||
m = make(map[string][]string)
|
||||
allMaps[id] = m
|
||||
return
|
||||
}
|
||||
|
||||
func deleteMap(id uint32) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
delete(allMaps, id)
|
||||
}
|
||||
|
||||
//export go_map_put_m4a_str
|
||||
func go_map_put_m4a_str(id C.ulong, key *C.char, val *C.char) {
|
||||
k := strings.ToLower(C.GoString(key))
|
||||
|
||||
// Special for M4A, do not catch keys that have no actual name
|
||||
k = strings.TrimPrefix(k, iTunesKeyPrefix)
|
||||
do_put_map(id, k, val)
|
||||
}
|
||||
|
||||
//export go_map_put_str
|
||||
func go_map_put_str(id C.ulong, key *C.char, val *C.char) {
|
||||
k := strings.ToLower(C.GoString(key))
|
||||
do_put_map(id, k, val)
|
||||
}
|
||||
|
||||
//export go_map_put_lyrics
|
||||
func go_map_put_lyrics(id C.ulong, lang *C.char, val *C.char) {
|
||||
k := "lyrics-" + strings.ToLower(C.GoString(lang))
|
||||
do_put_map(id, k, val)
|
||||
}
|
||||
|
||||
func do_put_map(id C.ulong, key string, val *C.char) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
m := allMaps[uint32(id)]
|
||||
v := strings.TrimSpace(C.GoString(val))
|
||||
m[key] = append(m[key], v)
|
||||
}
|
||||
|
||||
/*
|
||||
As I'm working on the new scanner, I see that the `properties` from TagLib is ill-suited to extract multi-valued ID3 frames. I'll have to change the way we do it for ID3, probably by sending the raw frames to Go and mapping there, instead of relying on the auto-mapped `properties`. I think this would reduce our reliance on C++, while also giving us more flexibility, including parsing the USLT / SYLT frames in Go
|
||||
*/
|
||||
|
||||
//export go_map_put_int
|
||||
func go_map_put_int(id C.ulong, key *C.char, val C.int) {
|
||||
valStr := strconv.Itoa(int(val))
|
||||
vp := C.CString(valStr)
|
||||
defer C.free(unsafe.Pointer(vp))
|
||||
go_map_put_str(id, key, vp)
|
||||
}
|
||||
|
||||
//export go_map_put_lyric_line
|
||||
func go_map_put_lyric_line(id C.ulong, lang *C.char, text *C.char, time C.int) {
|
||||
language := C.GoString(lang)
|
||||
line := C.GoString(text)
|
||||
timeGo := int64(time)
|
||||
|
||||
ms := timeGo % 1000
|
||||
timeGo /= 1000
|
||||
sec := timeGo % 60
|
||||
timeGo /= 60
|
||||
min := timeGo % 60
|
||||
formatted_line := fmt.Sprintf("[%02d:%02d.%02d]%s\n", min, sec, ms/10, line)
|
||||
|
||||
lock.RLock()
|
||||
defer lock.RUnlock()
|
||||
|
||||
key := "lyrics-" + language
|
||||
|
||||
m := allMaps[uint32(id)]
|
||||
existing, ok := m[key]
|
||||
if ok {
|
||||
existing[0] += formatted_line
|
||||
} else {
|
||||
m[key] = []string{formatted_line}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#define TAGLIB_ERR_PARSE -1
|
||||
#define TAGLIB_ERR_AUDIO_PROPS -2
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifdef WIN32
|
||||
#define FILENAME_CHAR_T wchar_t
|
||||
#else
|
||||
#define FILENAME_CHAR_T char
|
||||
#endif
|
||||
|
||||
extern void go_map_put_m4a_str(unsigned long id, char *key, char *val);
|
||||
extern void go_map_put_str(unsigned long id, char *key, char *val);
|
||||
extern void go_map_put_int(unsigned long id, char *key, int val);
|
||||
extern void go_map_put_lyrics(unsigned long id, char *lang, char *val);
|
||||
extern void go_map_put_lyric_line(unsigned long id, char *lang, char *text, int time);
|
||||
int taglib_read(const FILENAME_CHAR_T *filename, unsigned long id);
|
||||
char* taglib_version();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
"github.com/navidrome/navidrome/scanner/metadata_old"
|
||||
)
|
||||
|
||||
const ExtractorID = "ffmpeg"
|
||||
|
|
@ -20,13 +20,13 @@ type Extractor struct {
|
|||
ffmpeg ffmpeg.FFmpeg
|
||||
}
|
||||
|
||||
func (e *Extractor) Parse(files ...string) (map[string]metadata.ParsedTags, error) {
|
||||
func (e *Extractor) Parse(files ...string) (map[string]metadata_old.ParsedTags, error) {
|
||||
output, err := e.ffmpeg.Probe(context.TODO(), files)
|
||||
if err != nil {
|
||||
log.Error("Cannot use ffmpeg to extract tags. Aborting", err)
|
||||
return nil, err
|
||||
}
|
||||
fileTags := map[string]metadata.ParsedTags{}
|
||||
fileTags := map[string]metadata_old.ParsedTags{}
|
||||
if len(output) == 0 {
|
||||
return fileTags, errors.New("error extracting metadata files")
|
||||
}
|
||||
|
|
@ -41,8 +41,8 @@ func (e *Extractor) Parse(files ...string) (map[string]metadata.ParsedTags, erro
|
|||
return fileTags, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) CustomMappings() metadata.ParsedTags {
|
||||
return metadata.ParsedTags{
|
||||
func (e *Extractor) CustomMappings() metadata_old.ParsedTags {
|
||||
return metadata_old.ParsedTags{
|
||||
"disc": {"tpa"},
|
||||
"has_picture": {"metadata_block_picture"},
|
||||
"originaldate": {"tdor"},
|
||||
|
|
@ -53,7 +53,7 @@ func (e *Extractor) Version() string {
|
|||
return e.ffmpeg.Version()
|
||||
}
|
||||
|
||||
func (e *Extractor) extractMetadata(filePath, info string) (metadata.ParsedTags, error) {
|
||||
func (e *Extractor) extractMetadata(filePath, info string) (metadata_old.ParsedTags, error) {
|
||||
tags := e.parseInfo(info)
|
||||
if len(tags) == 0 {
|
||||
log.Trace("Not a media file. Skipping", "filePath", filePath)
|
||||
|
|
@ -207,5 +207,5 @@ func (e *Extractor) parseChannels(tag string) string {
|
|||
|
||||
// Inputs will always be absolute paths
|
||||
func init() {
|
||||
metadata.RegisterExtractor(ExtractorID, &Extractor{ffmpeg: ffmpeg.New()})
|
||||
metadata_old.RegisterExtractor(ExtractorID, &Extractor{ffmpeg: ffmpeg.New()})
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package metadata
|
||||
package metadata_old
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package metadata
|
||||
package metadata_old
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
|
|
@ -89,7 +89,7 @@ var _ = Describe("Tags", func() {
|
|||
})
|
||||
})
|
||||
|
||||
Describe("Bpm", func() {
|
||||
Describe("BPM", func() {
|
||||
var t *Tags
|
||||
BeforeEach(func() {
|
||||
t = &Tags{Tags: map[string][]string{
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package metadata
|
||||
package metadata_old
|
||||
|
||||
import (
|
||||
"testing"
|
||||
95
scanner/metadata_old/metadata_test.go
Normal file
95
scanner/metadata_old/metadata_test.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package metadata_old_test
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner/metadata_old"
|
||||
_ "github.com/navidrome/navidrome/scanner/metadata_old/ffmpeg"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Tags", func() {
|
||||
var zero int64 = 0
|
||||
var secondTs int64 = 2500
|
||||
|
||||
makeLyrics := func(synced bool, lang, secondLine string) model.Lyrics {
|
||||
lines := []model.Line{
|
||||
{Value: "This is"},
|
||||
{Value: secondLine},
|
||||
}
|
||||
|
||||
if synced {
|
||||
lines[0].Start = &zero
|
||||
lines[1].Start = &secondTs
|
||||
}
|
||||
|
||||
lyrics := model.Lyrics{
|
||||
Lang: lang,
|
||||
Line: lines,
|
||||
Synced: synced,
|
||||
}
|
||||
|
||||
return lyrics
|
||||
}
|
||||
|
||||
sortLyrics := func(lines model.LyricList) model.LyricList {
|
||||
slices.SortFunc(lines, func(a, b model.Lyrics) int {
|
||||
langDiff := cmp.Compare(a.Lang, b.Lang)
|
||||
if langDiff != 0 {
|
||||
return langDiff
|
||||
}
|
||||
return cmp.Compare(a.Line[1].Value, b.Line[1].Value)
|
||||
})
|
||||
|
||||
return lines
|
||||
}
|
||||
|
||||
compareLyrics := func(m metadata_old.Tags, expected model.LyricList) {
|
||||
lyrics := model.LyricList{}
|
||||
Expect(json.Unmarshal([]byte(m.Lyrics()), &lyrics)).To(BeNil())
|
||||
Expect(sortLyrics(lyrics)).To(Equal(sortLyrics(expected)))
|
||||
}
|
||||
|
||||
// Only run these tests if FFmpeg is available
|
||||
FFmpegContext := XContext
|
||||
if ffmpeg.New().IsAvailable() {
|
||||
FFmpegContext = Context
|
||||
}
|
||||
FFmpegContext("Extract with FFmpeg", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.Scanner.Extractor = "ffmpeg"
|
||||
})
|
||||
|
||||
DescribeTable("Lyrics test",
|
||||
func(file string) {
|
||||
path := "tests/fixtures/" + file
|
||||
mds, err := metadata_old.Extract(path)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mds).To(HaveLen(1))
|
||||
|
||||
m := mds[path]
|
||||
compareLyrics(m, model.LyricList{
|
||||
makeLyrics(true, "eng", "English"),
|
||||
makeLyrics(true, "xxx", "unspecified"),
|
||||
})
|
||||
},
|
||||
|
||||
Entry("Parses AIFF file", "test.aiff"),
|
||||
Entry("Parses MP3 files", "test.mp3"),
|
||||
// Disabled, because it fails in pipeline
|
||||
// Entry("Parses WAV files", "test.wav"),
|
||||
|
||||
// FFMPEG behaves very weirdly for multivalued tags for non-ID3
|
||||
// Specifically, they are separated by ";, which is indistinguishable
|
||||
// from other fields
|
||||
)
|
||||
})
|
||||
})
|
||||
471
scanner/phase_1_folders.go
Normal file
471
scanner/phase_1_folders.go
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/metadata"
|
||||
"github.com/navidrome/navidrome/utils"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer, libs []model.Library) *phaseFolders {
|
||||
var jobs []*scanJob
|
||||
for _, lib := range libs {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating last scan started at", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
// Reload library to get updated state
|
||||
l, err := ds.Library(ctx).Get(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
lib = *l
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
|
||||
}
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state}
|
||||
}
|
||||
|
||||
type scanJob struct {
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]time.Time
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
}
|
||||
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) {
|
||||
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting last updates: %w", err)
|
||||
}
|
||||
fileStore, err := storage.For(lib.Path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err)
|
||||
return nil, fmt.Errorf("getting storage for library: %w", err)
|
||||
}
|
||||
fsys, err := fileStore.FS()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting fs for library", "library", lib.Name, "path", lib.Path, err)
|
||||
return nil, fmt.Errorf("getting fs for library: %w", err)
|
||||
}
|
||||
lib.FullScanInProgress = lib.FullScanInProgress || fullScan
|
||||
return &scanJob{
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (j *scanJob) popLastUpdate(folderID string) time.Time {
|
||||
j.lock.Lock()
|
||||
defer j.lock.Unlock()
|
||||
|
||||
lastUpdate := j.lastUpdates[folderID]
|
||||
delete(j.lastUpdates, folderID)
|
||||
return lastUpdate
|
||||
}
|
||||
|
||||
// phaseFolders represents the first phase of the scanning process, which is responsible
|
||||
// for scanning all libraries and importing new or updated files. This phase involves
|
||||
// traversing the directory tree of each library, identifying new or modified media files,
|
||||
// and updating the database with the relevant information.
|
||||
//
|
||||
// The phaseFolders struct holds the context, data store, and jobs required for the scanning
|
||||
// process. Each job represents a library being scanned, and contains information about the
|
||||
// library, file system, and the last updates of the folders.
|
||||
//
|
||||
// The phaseFolders struct implements the phase interface, providing methods to produce
|
||||
// folder entries, process folders, persist changes to the database, and log the results.
|
||||
type phaseFolders struct {
|
||||
jobs []*scanJob
|
||||
ds model.DataStore
|
||||
ctx context.Context
|
||||
state *scanState
|
||||
prevAlbumPIDConf string
|
||||
}
|
||||
|
||||
func (p *phaseFolders) description() string {
|
||||
return "Scan all libraries and import new/updated files"
|
||||
}
|
||||
|
||||
func (p *phaseFolders) producer() ppl.Producer[*folderEntry] {
|
||||
return ppl.NewProducer(func(put func(entry *folderEntry)) error {
|
||||
var err error
|
||||
p.prevAlbumPIDConf, err = p.ds.Property(p.ctx).DefaultGet(consts.PIDAlbumKey, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting album PID conf: %w", err)
|
||||
}
|
||||
|
||||
// TODO Parallelize multiple job when we have multiple libraries
|
||||
var total int64
|
||||
var totalChanged int64
|
||||
for _, job := range p.jobs {
|
||||
if utils.IsCtxDone(p.ctx) {
|
||||
break
|
||||
}
|
||||
outputChan, err := walkDirTree(p.ctx, job)
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err)
|
||||
}
|
||||
for folder := range pl.ReadOrDone(p.ctx, outputChan) {
|
||||
job.numFolders.Add(1)
|
||||
p.state.sendProgress(&ProgressInfo{
|
||||
LibID: job.lib.ID,
|
||||
FileCount: uint32(len(folder.audioFiles)),
|
||||
Path: folder.path,
|
||||
Phase: "1",
|
||||
})
|
||||
if folder.isOutdated() {
|
||||
if !p.state.fullScan {
|
||||
if folder.hasNoFiles() && folder.isNew() {
|
||||
log.Trace(p.ctx, "Scanner: Skipping new folder with no files", "folder", folder.path, "lib", job.lib.Name)
|
||||
continue
|
||||
}
|
||||
log.Trace(p.ctx, "Scanner: Detected changes in folder", "folder", folder.path, "lastUpdate", folder.modTime, "lib", job.lib.Name)
|
||||
}
|
||||
totalChanged++
|
||||
folder.elapsed.Stop()
|
||||
put(folder)
|
||||
}
|
||||
}
|
||||
total += job.numFolders.Load()
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Finished loading all folders", "numFolders", total, "numChanged", totalChanged)
|
||||
return nil
|
||||
}, ppl.Name("traverse filesystem"))
|
||||
}
|
||||
|
||||
func (p *phaseFolders) measure(entry *folderEntry) func() time.Duration {
|
||||
entry.elapsed.Start()
|
||||
return func() time.Duration { return entry.elapsed.Stop() }
|
||||
}
|
||||
|
||||
func (p *phaseFolders) stages() []ppl.Stage[*folderEntry] {
|
||||
return []ppl.Stage[*folderEntry]{
|
||||
ppl.NewStage(p.processFolder, ppl.Name("process folder"), ppl.Concurrency(conf.Server.DevScannerThreads)),
|
||||
ppl.NewStage(p.persistChanges, ppl.Name("persist changes")),
|
||||
ppl.NewStage(p.logFolder, ppl.Name("log results")),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *phaseFolders) processFolder(entry *folderEntry) (*folderEntry, error) {
|
||||
defer p.measure(entry)()
|
||||
|
||||
// Load children mediafiles from DB
|
||||
cursor, err := p.ds.MediaFile(p.ctx).GetCursor(model.QueryOptions{
|
||||
Filters: squirrel.And{squirrel.Eq{"folder_id": entry.id}},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error loading mediafiles from DB", "folder", entry.path, err)
|
||||
return entry, err
|
||||
}
|
||||
dbTracks := make(map[string]*model.MediaFile)
|
||||
for mf, err := range cursor {
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error loading mediafiles from DB", "folder", entry.path, err)
|
||||
return entry, err
|
||||
}
|
||||
dbTracks[mf.Path] = &mf
|
||||
}
|
||||
|
||||
// Get list of files to import, based on modtime (or all if fullScan),
|
||||
// leave in dbTracks only tracks that are missing (not found in the FS)
|
||||
filesToImport := make(map[string]*model.MediaFile, len(entry.audioFiles))
|
||||
for afPath, af := range entry.audioFiles {
|
||||
fullPath := path.Join(entry.path, afPath)
|
||||
dbTrack, foundInDB := dbTracks[fullPath]
|
||||
if !foundInDB || p.state.fullScan {
|
||||
filesToImport[fullPath] = dbTrack
|
||||
} else {
|
||||
info, err := af.Info()
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error getting file info", "folder", entry.path, "file", af.Name(), err)
|
||||
p.state.sendWarning(fmt.Sprintf("Error getting file info for %s/%s: %v", entry.path, af.Name(), err))
|
||||
return entry, nil
|
||||
}
|
||||
if info.ModTime().After(dbTrack.UpdatedAt) || dbTrack.Missing {
|
||||
filesToImport[fullPath] = dbTrack
|
||||
}
|
||||
}
|
||||
delete(dbTracks, fullPath)
|
||||
}
|
||||
|
||||
// Remaining dbTracks are tracks that were not found in the FS, so they should be marked as missing
|
||||
entry.missingTracks = slices.Collect(maps.Values(dbTracks))
|
||||
|
||||
// Load metadata from files that need to be imported
|
||||
if len(filesToImport) > 0 {
|
||||
err = p.loadTagsFromFiles(entry, filesToImport)
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error loading tags from files. Skipping", "folder", entry.path, err)
|
||||
p.state.sendWarning(fmt.Sprintf("Error loading tags from files in %s: %v", entry.path, err))
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
p.createAlbumsFromMediaFiles(entry)
|
||||
p.createArtistsFromMediaFiles(entry)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
const filesBatchSize = 200
|
||||
|
||||
// loadTagsFromFiles reads metadata from the files in the given list and populates
|
||||
// the entry's tracks and tags with the results.
|
||||
func (p *phaseFolders) loadTagsFromFiles(entry *folderEntry, toImport map[string]*model.MediaFile) error {
|
||||
tracks := make([]model.MediaFile, 0, len(toImport))
|
||||
uniqueTags := make(map[string]model.Tag, len(toImport))
|
||||
for chunk := range slice.CollectChunks(maps.Keys(toImport), filesBatchSize) {
|
||||
allInfo, err := entry.job.fs.ReadTags(chunk...)
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error extracting metadata from files. Skipping", "folder", entry.path, err)
|
||||
return err
|
||||
}
|
||||
for filePath, info := range allInfo {
|
||||
md := metadata.New(filePath, info)
|
||||
track := md.ToMediaFile(entry.job.lib.ID, entry.id)
|
||||
tracks = append(tracks, track)
|
||||
for _, t := range track.Tags.FlattenAll() {
|
||||
uniqueTags[t.ID] = t
|
||||
}
|
||||
|
||||
// Keep track of any album ID changes, to reassign annotations later
|
||||
prevAlbumID := ""
|
||||
if prev := toImport[filePath]; prev != nil {
|
||||
prevAlbumID = prev.AlbumID
|
||||
} else {
|
||||
prevAlbumID = md.AlbumID(track, p.prevAlbumPIDConf)
|
||||
}
|
||||
_, ok := entry.albumIDMap[track.AlbumID]
|
||||
if prevAlbumID != track.AlbumID && !ok {
|
||||
entry.albumIDMap[track.AlbumID] = prevAlbumID
|
||||
}
|
||||
}
|
||||
}
|
||||
entry.tracks = tracks
|
||||
entry.tags = slices.Collect(maps.Values(uniqueTags))
|
||||
return nil
|
||||
}
|
||||
|
||||
// createAlbumsFromMediaFiles groups the entry's tracks by album ID and creates albums
|
||||
func (p *phaseFolders) createAlbumsFromMediaFiles(entry *folderEntry) {
|
||||
grouped := slice.Group(entry.tracks, func(mf model.MediaFile) string { return mf.AlbumID })
|
||||
albums := make(model.Albums, 0, len(grouped))
|
||||
for _, group := range grouped {
|
||||
songs := model.MediaFiles(group)
|
||||
album := songs.ToAlbum()
|
||||
albums = append(albums, album)
|
||||
}
|
||||
entry.albums = albums
|
||||
}
|
||||
|
||||
// createArtistsFromMediaFiles creates artists from the entry's tracks
|
||||
func (p *phaseFolders) createArtistsFromMediaFiles(entry *folderEntry) {
|
||||
participants := make(model.Participants, len(entry.tracks)*3) // preallocate ~3 artists per track
|
||||
for _, track := range entry.tracks {
|
||||
participants.Merge(track.Participants)
|
||||
}
|
||||
entry.artists = participants.AllArtists()
|
||||
}
|
||||
|
||||
func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) {
|
||||
defer p.measure(entry)()
|
||||
p.state.changesDetected.Store(true)
|
||||
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
// Instantiate all repositories just once per folder
|
||||
folderRepo := tx.Folder(p.ctx)
|
||||
tagRepo := tx.Tag(p.ctx)
|
||||
artistRepo := tx.Artist(p.ctx)
|
||||
libraryRepo := tx.Library(p.ctx)
|
||||
albumRepo := tx.Album(p.ctx)
|
||||
mfRepo := tx.MediaFile(p.ctx)
|
||||
|
||||
// Save folder to DB
|
||||
folder := entry.toFolder()
|
||||
err := folderRepo.Put(folder)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting folder to DB", "folder", entry.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Save all tags to DB
|
||||
err = tagRepo.Add(entry.tags...)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting tags to DB", "folder", entry.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Save all new/modified artists to DB. Their information will be incomplete, but they will be refreshed later
|
||||
for i := range entry.artists {
|
||||
err = artistRepo.Put(&entry.artists[i], "name", "mbz_artist_id", "sort_artist_name", "order_artist_name")
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting artist to DB", "folder", entry.path, "artist", entry.artists[i].Name, err)
|
||||
return err
|
||||
}
|
||||
err = libraryRepo.AddArtist(entry.job.lib.ID, entry.artists[i].ID)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error adding artist to library", "lib", entry.job.lib.ID, "artist", entry.artists[i].Name, err)
|
||||
return err
|
||||
}
|
||||
if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists {
|
||||
entry.job.cw.PreCache(entry.artists[i].CoverArtID())
|
||||
}
|
||||
}
|
||||
|
||||
// Save all new/modified albums to DB. Their information will be incomplete, but they will be refreshed later
|
||||
for i := range entry.albums {
|
||||
err = p.persistAlbum(albumRepo, &entry.albums[i], entry.albumIDMap)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting album to DB", "folder", entry.path, "album", entry.albums[i], err)
|
||||
return err
|
||||
}
|
||||
if entry.albums[i].Name != consts.UnknownAlbum {
|
||||
entry.job.cw.PreCache(entry.albums[i].CoverArtID())
|
||||
}
|
||||
}
|
||||
|
||||
// Save all tracks to DB
|
||||
for i := range entry.tracks {
|
||||
err = mfRepo.Put(&entry.tracks[i])
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting mediafile to DB", "folder", entry.path, "track", entry.tracks[i], err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Mark all missing tracks as not available
|
||||
if len(entry.missingTracks) > 0 {
|
||||
err = mfRepo.MarkMissing(true, entry.missingTracks...)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error marking missing tracks", "folder", entry.path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Touch all albums that have missing tracks, so they get refreshed in later phases
|
||||
groupedMissingTracks := slice.ToMap(entry.missingTracks, func(mf *model.MediaFile) (string, struct{}) {
|
||||
return mf.AlbumID, struct{}{}
|
||||
})
|
||||
albumsToUpdate := slices.Collect(maps.Keys(groupedMissingTracks))
|
||||
err = albumRepo.Touch(albumsToUpdate...)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error touching album", "folder", entry.path, "albums", albumsToUpdate, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
|
||||
}
|
||||
return entry, err
|
||||
}
|
||||
|
||||
// persistAlbum persists the given album to the database, and reassigns annotations from the previous album ID
|
||||
func (p *phaseFolders) persistAlbum(repo model.AlbumRepository, a *model.Album, idMap map[string]string) error {
|
||||
prevID := idMap[a.ID]
|
||||
log.Trace(p.ctx, "Persisting album", "album", a.Name, "albumArtist", a.AlbumArtist, "id", a.ID, "prevID", cmp.Or(prevID, "nil"))
|
||||
if err := repo.Put(a); err != nil {
|
||||
return fmt.Errorf("persisting album %s: %w", a.ID, err)
|
||||
}
|
||||
if prevID == "" {
|
||||
return nil
|
||||
}
|
||||
// Reassign annotation from previous album to new album
|
||||
log.Trace(p.ctx, "Reassigning album annotations", "from", prevID, "to", a.ID, "album", a.Name)
|
||||
if err := repo.ReassignAnnotation(prevID, a.ID); err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Could not reassign annotations", "from", prevID, "to", a.ID, "album", a.Name, err)
|
||||
p.state.sendWarning(fmt.Sprintf("Could not reassign annotations from %s to %s ('%s'): %v", prevID, a.ID, a.Name, err))
|
||||
}
|
||||
// Keep created_at field from previous instance of the album
|
||||
if err := repo.CopyAttributes(prevID, a.ID, "created_at"); err != nil {
|
||||
// Silently ignore when the previous album is not found
|
||||
if !errors.Is(err, model.ErrNotFound) {
|
||||
log.Warn(p.ctx, "Scanner: Could not copy fields", "from", prevID, "to", a.ID, "album", a.Name, err)
|
||||
p.state.sendWarning(fmt.Sprintf("Could not copy fields from %s to %s ('%s'): %v", prevID, a.ID, a.Name, err))
|
||||
}
|
||||
}
|
||||
// Don't keep track of this mapping anymore
|
||||
delete(idMap, a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *phaseFolders) logFolder(entry *folderEntry) (*folderEntry, error) {
|
||||
logCall := log.Info
|
||||
if entry.hasNoFiles() {
|
||||
logCall = log.Trace
|
||||
}
|
||||
logCall(p.ctx, "Scanner: Completed processing folder",
|
||||
"audioCount", len(entry.audioFiles), "imageCount", len(entry.imageFiles), "plsCount", entry.numPlaylists,
|
||||
"elapsed", entry.elapsed.Elapsed(), "tracksMissing", len(entry.missingTracks),
|
||||
"tracksImported", len(entry.tracks), "library", entry.job.lib.Name, consts.Zwsp+"folder", entry.path)
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (p *phaseFolders) finalize(err error) error {
|
||||
errF := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
for _, job := range p.jobs {
|
||||
// Mark all folders that were not updated as missing
|
||||
if len(job.lastUpdates) == 0 {
|
||||
continue
|
||||
}
|
||||
folderIDs := slices.Collect(maps.Keys(job.lastUpdates))
|
||||
err := tx.Folder(p.ctx).MarkMissing(true, folderIDs...)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error marking missing folders", "lib", job.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
err = tx.MediaFile(p.ctx).MarkMissingByFolder(true, folderIDs...)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error marking tracks in missing folders", "lib", job.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
// Touch all albums that have missing folders, so they get refreshed in later phases
|
||||
_, err = tx.Album(p.ctx).TouchByMissingFolder()
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error touching albums with missing folders", "lib", job.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return errors.Join(err, errF)
|
||||
}
|
||||
|
||||
var _ phase[*folderEntry] = (*phaseFolders)(nil)
|
||||
192
scanner/phase_2_missing_tracks.go
Normal file
192
scanner/phase_2_missing_tracks.go
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type missingTracks struct {
|
||||
lib model.Library
|
||||
pid string
|
||||
missing model.MediaFiles
|
||||
matched model.MediaFiles
|
||||
}
|
||||
|
||||
// phaseMissingTracks is responsible for processing missing media files during the scan process.
|
||||
// It identifies media files that are marked as missing and attempts to find matching files that
|
||||
// may have been moved or renamed. This phase helps in maintaining the integrity of the media
|
||||
// library by ensuring that moved or renamed files are correctly updated in the database.
|
||||
//
|
||||
// The phaseMissingTracks phase performs the following steps:
|
||||
// 1. Loads all libraries and their missing media files from the database.
|
||||
// 2. For each library, it sorts the missing files by their PID (persistent identifier).
|
||||
// 3. Groups missing and matched files by their PID and processes them to find exact or equivalent matches.
|
||||
// 4. Updates the database with the new locations of the matched files and removes the old entries.
|
||||
// 5. Logs the results and finalizes the phase by reporting the total number of matched files.
|
||||
type phaseMissingTracks struct {
|
||||
ctx context.Context
|
||||
ds model.DataStore
|
||||
totalMatched atomic.Uint32
|
||||
state *scanState
|
||||
}
|
||||
|
||||
func createPhaseMissingTracks(ctx context.Context, state *scanState, ds model.DataStore) *phaseMissingTracks {
|
||||
return &phaseMissingTracks{ctx: ctx, ds: ds, state: state}
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) description() string {
|
||||
return "Process missing files, checking for moves"
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) producer() ppl.Producer[*missingTracks] {
|
||||
return ppl.NewProducer(p.produce, ppl.Name("load missing tracks from db"))
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) produce(put func(tracks *missingTracks)) error {
|
||||
count := 0
|
||||
var putIfMatched = func(mt missingTracks) {
|
||||
if mt.pid != "" && len(mt.matched) > 0 {
|
||||
log.Trace(p.ctx, "Scanner: Found missing and matching tracks", "pid", mt.pid, "missing", len(mt.missing), "matched", len(mt.matched), "lib", mt.lib.Name)
|
||||
count++
|
||||
put(&mt)
|
||||
}
|
||||
}
|
||||
libs, err := p.ds.Library(p.ctx).GetAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading libraries: %w", err)
|
||||
}
|
||||
for _, lib := range libs {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Checking missing tracks", "libraryId", lib.ID, "libraryName", lib.Name)
|
||||
cursor, err := p.ds.MediaFile(p.ctx).GetMissingAndMatching(lib.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading missing tracks for library %s: %w", lib.Name, err)
|
||||
}
|
||||
|
||||
// Group missing and matched tracks by PID
|
||||
mt := missingTracks{lib: lib}
|
||||
for mf, err := range cursor {
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading missing tracks for library %s: %w", lib.Name, err)
|
||||
}
|
||||
if mt.pid != mf.PID {
|
||||
putIfMatched(mt)
|
||||
mt.pid = mf.PID
|
||||
mt.missing = nil
|
||||
mt.matched = nil
|
||||
}
|
||||
if mf.Missing {
|
||||
mt.missing = append(mt.missing, mf)
|
||||
} else {
|
||||
mt.matched = append(mt.matched, mf)
|
||||
}
|
||||
}
|
||||
putIfMatched(mt)
|
||||
if count == 0 {
|
||||
log.Debug(p.ctx, "Scanner: No potential moves found", "libraryId", lib.ID, "libraryName", lib.Name)
|
||||
} else {
|
||||
log.Debug(p.ctx, "Scanner: Found potential moves", "libraryId", lib.ID, "count", count)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) stages() []ppl.Stage[*missingTracks] {
|
||||
return []ppl.Stage[*missingTracks]{
|
||||
ppl.NewStage(p.processMissingTracks, ppl.Name("process missing tracks")),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) processMissingTracks(in *missingTracks) (*missingTracks, error) {
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
for _, ms := range in.missing {
|
||||
var exactMatch model.MediaFile
|
||||
var equivalentMatch model.MediaFile
|
||||
|
||||
// Identify exact and equivalent matches
|
||||
for _, mt := range in.matched {
|
||||
if ms.Equals(mt) {
|
||||
exactMatch = mt
|
||||
break // Prioritize exact match
|
||||
}
|
||||
if ms.IsEquivalent(mt) {
|
||||
equivalentMatch = mt
|
||||
}
|
||||
}
|
||||
|
||||
// Use the exact match if found
|
||||
if exactMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track in a new place", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, exactMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error moving matched track", "missing", ms.Path, "movedTo", exactMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// If there is only one missing and one matched track, consider them equivalent (same PID)
|
||||
if len(in.missing) == 1 && len(in.matched) == 1 {
|
||||
singleMatch := in.matched[0]
|
||||
log.Debug(p.ctx, "Scanner: Found track with same persistent ID in a new place", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, singleMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", singleMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
continue
|
||||
}
|
||||
|
||||
// Use the equivalent match if no other better match was found
|
||||
if equivalentMatch.ID != "" {
|
||||
log.Debug(p.ctx, "Scanner: Found missing track with same base path", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name)
|
||||
err := p.moveMatched(tx, equivalentMatch, ms)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error updating matched track", "missing", ms.Path, "movedTo", equivalentMatch.Path, "lib", in.lib.Name, err)
|
||||
return err
|
||||
}
|
||||
p.totalMatched.Add(1)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) moveMatched(tx model.DataStore, mt, ms model.MediaFile) error {
|
||||
discardedID := mt.ID
|
||||
mt.ID = ms.ID
|
||||
err := tx.MediaFile(p.ctx).Put(&mt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update matched track: %w", err)
|
||||
}
|
||||
err = tx.MediaFile(p.ctx).Delete(discardedID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete discarded track: %w", err)
|
||||
}
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *phaseMissingTracks) finalize(err error) error {
|
||||
matched := p.totalMatched.Load()
|
||||
if matched > 0 {
|
||||
log.Info(p.ctx, "Scanner: Found moved files", "total", matched, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var _ phase[*missingTracks] = (*phaseMissingTracks)(nil)
|
||||
225
scanner/phase_2_missing_tracks_test.go
Normal file
225
scanner/phase_2_missing_tracks_test.go
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("phaseMissingTracks", func() {
|
||||
var (
|
||||
phase *phaseMissingTracks
|
||||
ctx context.Context
|
||||
ds model.DataStore
|
||||
mr *tests.MockMediaFileRepo
|
||||
lr *tests.MockLibraryRepo
|
||||
state *scanState
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
mr = tests.CreateMockMediaFileRepo()
|
||||
lr = &tests.MockLibraryRepo{}
|
||||
lr.SetData(model.Libraries{{ID: 1, LastScanStartedAt: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)}})
|
||||
ds = &tests.MockDataStore{MockedMediaFile: mr, MockedLibrary: lr}
|
||||
state = &scanState{}
|
||||
phase = createPhaseMissingTracks(ctx, state, ds)
|
||||
})
|
||||
|
||||
Describe("produceMissingTracks", func() {
|
||||
var (
|
||||
put func(tracks *missingTracks)
|
||||
produced []*missingTracks
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
produced = nil
|
||||
put = func(tracks *missingTracks) {
|
||||
produced = append(produced, tracks)
|
||||
}
|
||||
})
|
||||
|
||||
When("there are no missing tracks", func() {
|
||||
It("should not call put", func() {
|
||||
mr.SetData(model.MediaFiles{
|
||||
{ID: "1", PID: "A", Missing: false},
|
||||
{ID: "2", PID: "A", Missing: false},
|
||||
})
|
||||
|
||||
err := phase.produce(put)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(produced).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
When("there are missing tracks", func() {
|
||||
It("should call put for any missing tracks with corresponding matches", func() {
|
||||
mr.SetData(model.MediaFiles{
|
||||
{ID: "1", PID: "A", Missing: true, LibraryID: 1},
|
||||
{ID: "2", PID: "B", Missing: true, LibraryID: 1},
|
||||
{ID: "3", PID: "A", Missing: false, LibraryID: 1},
|
||||
})
|
||||
|
||||
err := phase.produce(put)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(produced).To(HaveLen(1))
|
||||
Expect(produced[0].pid).To(Equal("A"))
|
||||
Expect(produced[0].missing).To(HaveLen(1))
|
||||
Expect(produced[0].matched).To(HaveLen(1))
|
||||
})
|
||||
It("should not call put if there are no matches for any missing tracks", func() {
|
||||
mr.SetData(model.MediaFiles{
|
||||
{ID: "1", PID: "A", Missing: true, LibraryID: 1},
|
||||
{ID: "2", PID: "B", Missing: true, LibraryID: 1},
|
||||
{ID: "3", PID: "C", Missing: false, LibraryID: 1},
|
||||
})
|
||||
|
||||
err := phase.produce(put)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(produced).To(BeZero())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("processMissingTracks", func() {
|
||||
It("should move the matched track when the missing track is the exact same", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "dir1/path1.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
matchedTrack := model.MediaFile{ID: "2", PID: "A", Path: "dir2/path2.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedTrack)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
matched: []model.MediaFile{matchedTrack},
|
||||
}
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(1)))
|
||||
Expect(state.changesDetected.Load()).To(BeTrue())
|
||||
|
||||
movedTrack, _ := ds.MediaFile(ctx).Get("1")
|
||||
Expect(movedTrack.Path).To(Equal(matchedTrack.Path))
|
||||
})
|
||||
|
||||
It("should move the matched track when the missing track has the same tags and filename", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "path1.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
matchedTrack := model.MediaFile{ID: "2", PID: "A", Path: "path1.flac", Tags: model.Tags{"title": []string{"title1"}}, Size: 200}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedTrack)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
matched: []model.MediaFile{matchedTrack},
|
||||
}
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(1)))
|
||||
Expect(state.changesDetected.Load()).To(BeTrue())
|
||||
|
||||
movedTrack, _ := ds.MediaFile(ctx).Get("1")
|
||||
Expect(movedTrack.Path).To(Equal(matchedTrack.Path))
|
||||
Expect(movedTrack.Size).To(Equal(matchedTrack.Size))
|
||||
})
|
||||
|
||||
It("should move the matched track when there's only one missing track and one matched track (same PID)", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "dir1/path1.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
matchedTrack := model.MediaFile{ID: "2", PID: "A", Path: "dir2/path2.flac", Tags: model.Tags{"title": []string{"different title"}}, Size: 200}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedTrack)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
matched: []model.MediaFile{matchedTrack},
|
||||
}
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(1)))
|
||||
Expect(state.changesDetected.Load()).To(BeTrue())
|
||||
|
||||
movedTrack, _ := ds.MediaFile(ctx).Get("1")
|
||||
Expect(movedTrack.Path).To(Equal(matchedTrack.Path))
|
||||
Expect(movedTrack.Size).To(Equal(matchedTrack.Size))
|
||||
})
|
||||
|
||||
It("should prioritize exact matches", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "dir1/file1.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
matchedEquivalent := model.MediaFile{ID: "2", PID: "A", Path: "dir1/file1.flac", Tags: model.Tags{"title": []string{"title1"}}, Size: 200}
|
||||
matchedExact := model.MediaFile{ID: "3", PID: "A", Path: "dir2/file2.mp3", Tags: model.Tags{"title": []string{"title1"}}, Size: 100}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedEquivalent)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedExact)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
// Note that equivalent comes before the exact match
|
||||
matched: []model.MediaFile{matchedEquivalent, matchedExact},
|
||||
}
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(1)))
|
||||
Expect(state.changesDetected.Load()).To(BeTrue())
|
||||
|
||||
movedTrack, _ := ds.MediaFile(ctx).Get("1")
|
||||
Expect(movedTrack.Path).To(Equal(matchedExact.Path))
|
||||
Expect(movedTrack.Size).To(Equal(matchedExact.Size))
|
||||
})
|
||||
|
||||
It("should not move anything if there's more than one match and they don't are not exact nor equivalent", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "dir1/file1.mp3", Title: "title1", Size: 100}
|
||||
matched1 := model.MediaFile{ID: "2", PID: "A", Path: "dir1/file2.flac", Title: "another title", Size: 200}
|
||||
matched2 := model.MediaFile{ID: "3", PID: "A", Path: "dir2/file3.mp3", Title: "different title", Size: 100}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matched1)
|
||||
_ = ds.MediaFile(ctx).Put(&matched2)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
matched: []model.MediaFile{matched1, matched2},
|
||||
}
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(phase.totalMatched.Load()).To(Equal(uint32(0)))
|
||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||
|
||||
// The missing track should still be the same
|
||||
movedTrack, _ := ds.MediaFile(ctx).Get("1")
|
||||
Expect(movedTrack.Path).To(Equal(missingTrack.Path))
|
||||
Expect(movedTrack.Title).To(Equal(missingTrack.Title))
|
||||
Expect(movedTrack.Size).To(Equal(missingTrack.Size))
|
||||
})
|
||||
|
||||
It("should return an error when there's an error moving the matched track", func() {
|
||||
missingTrack := model.MediaFile{ID: "1", PID: "A", Path: "path1.mp3", Tags: model.Tags{"title": []string{"title1"}}}
|
||||
matchedTrack := model.MediaFile{ID: "2", PID: "A", Path: "path1.mp3", Tags: model.Tags{"title": []string{"title1"}}}
|
||||
|
||||
_ = ds.MediaFile(ctx).Put(&missingTrack)
|
||||
_ = ds.MediaFile(ctx).Put(&matchedTrack)
|
||||
|
||||
in := &missingTracks{
|
||||
missing: []model.MediaFile{missingTrack},
|
||||
matched: []model.MediaFile{matchedTrack},
|
||||
}
|
||||
|
||||
// Simulate an error when moving the matched track by deleting the track from the DB
|
||||
_ = ds.MediaFile(ctx).Delete("2")
|
||||
|
||||
_, err := phase.processMissingTracks(in)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
157
scanner/phase_3_refresh_albums.go
Normal file
157
scanner/phase_3_refresh_albums.go
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
// nolint:unused
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// phaseRefreshAlbums is responsible for refreshing albums that have been
|
||||
// newly added or changed during the scan process. This phase ensures that
|
||||
// the album information in the database is up-to-date by performing the
|
||||
// following steps:
|
||||
// 1. Loads all libraries and their albums that have been touched (new or changed).
|
||||
// 2. For each album, it filters out unmodified albums by comparing the current
|
||||
// state with the state in the database.
|
||||
// 3. Refreshes the album information in the database if any changes are detected.
|
||||
// 4. Logs the results and finalizes the phase by reporting the total number of
|
||||
// refreshed and skipped albums.
|
||||
// 5. As a last step, it refreshes the artist statistics to reflect the changes
|
||||
type phaseRefreshAlbums struct {
|
||||
ds model.DataStore
|
||||
ctx context.Context
|
||||
libs model.Libraries
|
||||
refreshed atomic.Uint32
|
||||
skipped atomic.Uint32
|
||||
state *scanState
|
||||
}
|
||||
|
||||
func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore, libs model.Libraries) *phaseRefreshAlbums {
|
||||
return &phaseRefreshAlbums{ctx: ctx, ds: ds, libs: libs, state: state}
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) description() string {
|
||||
return "Refresh all new/changed albums"
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) producer() ppl.Producer[*model.Album] {
|
||||
return ppl.NewProducer(p.produce, ppl.Name("load albums from db"))
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) produce(put func(album *model.Album)) error {
|
||||
count := 0
|
||||
for _, lib := range p.libs {
|
||||
cursor, err := p.ds.Album(p.ctx).GetTouchedAlbums(lib.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading touched albums: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Checking albums that may need refresh", "libraryId", lib.ID, "libraryName", lib.Name)
|
||||
for album, err := range cursor {
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading touched albums: %w", err)
|
||||
}
|
||||
count++
|
||||
put(&album)
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
log.Debug(p.ctx, "Scanner: No albums needing refresh")
|
||||
} else {
|
||||
log.Debug(p.ctx, "Scanner: Found albums that may need refreshing", "count", count)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) stages() []ppl.Stage[*model.Album] {
|
||||
return []ppl.Stage[*model.Album]{
|
||||
ppl.NewStage(p.filterUnmodified, ppl.Name("filter unmodified"), ppl.Concurrency(5)),
|
||||
ppl.NewStage(p.refreshAlbum, ppl.Name("refresh albums")),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) filterUnmodified(album *model.Album) (*model.Album, error) {
|
||||
mfs, err := p.ds.MediaFile(p.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": album.ID}})
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Error loading media files for album", "album_id", album.ID, err)
|
||||
return nil, err
|
||||
}
|
||||
if len(mfs) == 0 {
|
||||
log.Debug(p.ctx, "Scanner: album has no media files. Skipping", "album_id", album.ID,
|
||||
"name", album.Name, "songCount", album.SongCount, "updatedAt", album.UpdatedAt)
|
||||
p.skipped.Add(1)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
newAlbum := mfs.ToAlbum()
|
||||
if album.Equals(newAlbum) {
|
||||
log.Trace("Scanner: album is up to date. Skipping", "album_id", album.ID,
|
||||
"name", album.Name, "songCount", album.SongCount, "updatedAt", album.UpdatedAt)
|
||||
p.skipped.Add(1)
|
||||
return nil, nil
|
||||
}
|
||||
return &newAlbum, nil
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) refreshAlbum(album *model.Album) (*model.Album, error) {
|
||||
if album == nil {
|
||||
return nil, nil
|
||||
}
|
||||
start := time.Now()
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
err := tx.Album(p.ctx).Put(album)
|
||||
log.Debug(p.ctx, "Scanner: refreshing album", "album_id", album.ID, "name", album.Name, "songCount", album.SongCount, "elapsed", time.Since(start))
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing album %s: %w", album.ID, err)
|
||||
}
|
||||
p.refreshed.Add(1)
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return album, nil
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) finalize(err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logF := log.Info
|
||||
refreshed := p.refreshed.Load()
|
||||
skipped := p.skipped.Load()
|
||||
if refreshed == 0 {
|
||||
logF = log.Debug
|
||||
}
|
||||
logF(p.ctx, "Scanner: Finished refreshing albums", "refreshed", refreshed, "skipped", skipped, err)
|
||||
if !p.state.changesDetected.Load() {
|
||||
log.Debug(p.ctx, "Scanner: No changes detected, skipping refreshing annotations")
|
||||
return nil
|
||||
}
|
||||
return p.ds.WithTx(func(tx model.DataStore) error {
|
||||
// Refresh album annotations
|
||||
start := time.Now()
|
||||
cnt, err := tx.Album(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing album annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed album annotations", "albums", cnt, "elapsed", time.Since(start))
|
||||
|
||||
// Refresh artist annotations
|
||||
start = time.Now()
|
||||
cnt, err = tx.Artist(p.ctx).RefreshPlayCounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("refreshing artist annotations: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Refreshed artist annotations", "artists", cnt, "elapsed", time.Since(start))
|
||||
p.state.changesDetected.Store(true)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
135
scanner/phase_3_refresh_albums_test.go
Normal file
135
scanner/phase_3_refresh_albums_test.go
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("phaseRefreshAlbums", func() {
|
||||
var (
|
||||
phase *phaseRefreshAlbums
|
||||
ctx context.Context
|
||||
albumRepo *tests.MockAlbumRepo
|
||||
mfRepo *tests.MockMediaFileRepo
|
||||
ds *tests.MockDataStore
|
||||
libs model.Libraries
|
||||
state *scanState
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
albumRepo = tests.CreateMockAlbumRepo()
|
||||
mfRepo = tests.CreateMockMediaFileRepo()
|
||||
ds = &tests.MockDataStore{
|
||||
MockedAlbum: albumRepo,
|
||||
MockedMediaFile: mfRepo,
|
||||
}
|
||||
libs = model.Libraries{
|
||||
{ID: 1, Name: "Library 1"},
|
||||
{ID: 2, Name: "Library 2"},
|
||||
}
|
||||
state = &scanState{}
|
||||
phase = createPhaseRefreshAlbums(ctx, state, ds, libs)
|
||||
})
|
||||
|
||||
Describe("description", func() {
|
||||
It("returns the correct description", func() {
|
||||
Expect(phase.description()).To(Equal("Refresh all new/changed albums"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("producer", func() {
|
||||
It("produces albums that need refreshing", func() {
|
||||
albumRepo.SetData(model.Albums{
|
||||
{LibraryID: 1, ID: "album1", Name: "Album 1"},
|
||||
})
|
||||
|
||||
var produced []*model.Album
|
||||
err := phase.produce(func(album *model.Album) {
|
||||
produced = append(produced, album)
|
||||
})
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(produced).To(HaveLen(1))
|
||||
Expect(produced[0].ID).To(Equal("album1"))
|
||||
})
|
||||
|
||||
It("returns an error if there is an error loading albums", func() {
|
||||
albumRepo.SetData(model.Albums{
|
||||
{ID: "error"},
|
||||
})
|
||||
|
||||
err := phase.produce(func(album *model.Album) {})
|
||||
|
||||
Expect(err).To(MatchError(ContainSubstring("loading touched albums")))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("filterUnmodified", func() {
|
||||
It("filters out unmodified albums", func() {
|
||||
album := &model.Album{ID: "album1", Name: "Album 1", SongCount: 1,
|
||||
FolderIDs: []string{"folder1"}, Discs: model.Discs{1: ""}}
|
||||
mfRepo.SetData(model.MediaFiles{
|
||||
{AlbumID: "album1", Title: "Song 1", Album: "Album 1", FolderID: "folder1"},
|
||||
})
|
||||
|
||||
result, err := phase.filterUnmodified(album)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(result).To(BeNil())
|
||||
})
|
||||
It("keep modified albums", func() {
|
||||
album := &model.Album{ID: "album1", Name: "Album 1"}
|
||||
mfRepo.SetData(model.MediaFiles{
|
||||
{AlbumID: "album1", Title: "Song 1", Album: "Album 2"},
|
||||
})
|
||||
|
||||
result, err := phase.filterUnmodified(album)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(result).ToNot(BeNil())
|
||||
Expect(result.ID).To(Equal("album1"))
|
||||
})
|
||||
It("skips albums with no media files", func() {
|
||||
album := &model.Album{ID: "album1", Name: "Album 1"}
|
||||
mfRepo.SetData(model.MediaFiles{})
|
||||
|
||||
result, err := phase.filterUnmodified(album)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(result).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("refreshAlbum", func() {
|
||||
It("refreshes the album in the database", func() {
|
||||
Expect(albumRepo.CountAll()).To(Equal(int64(0)))
|
||||
|
||||
album := &model.Album{ID: "album1", Name: "Album 1"}
|
||||
result, err := phase.refreshAlbum(album)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(result).ToNot(BeNil())
|
||||
Expect(result.ID).To(Equal("album1"))
|
||||
|
||||
savedAlbum, err := albumRepo.Get("album1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(savedAlbum).ToNot(BeNil())
|
||||
Expect(savedAlbum.ID).To(Equal("album1"))
|
||||
Expect(phase.refreshed.Load()).To(Equal(uint32(1)))
|
||||
Expect(state.changesDetected.Load()).To(BeTrue())
|
||||
})
|
||||
|
||||
It("returns an error if there is an error refreshing the album", func() {
|
||||
album := &model.Album{ID: "album1", Name: "Album 1"}
|
||||
albumRepo.SetError(true)
|
||||
|
||||
result, err := phase.refreshAlbum(album)
|
||||
Expect(result).To(BeNil())
|
||||
Expect(err).To(MatchError(ContainSubstring("refreshing album")))
|
||||
Expect(phase.refreshed.Load()).To(Equal(uint32(0)))
|
||||
Expect(state.changesDetected.Load()).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
126
scanner/phase_4_playlists.go
Normal file
126
scanner/phase_4_playlists.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
)
|
||||
|
||||
type phasePlaylists struct {
|
||||
ctx context.Context
|
||||
scanState *scanState
|
||||
ds model.DataStore
|
||||
pls core.Playlists
|
||||
cw artwork.CacheWarmer
|
||||
refreshed atomic.Uint32
|
||||
}
|
||||
|
||||
func createPhasePlaylists(ctx context.Context, scanState *scanState, ds model.DataStore, pls core.Playlists, cw artwork.CacheWarmer) *phasePlaylists {
|
||||
return &phasePlaylists{
|
||||
ctx: ctx,
|
||||
scanState: scanState,
|
||||
ds: ds,
|
||||
pls: pls,
|
||||
cw: cw,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) description() string {
|
||||
return "Import/update playlists"
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) producer() ppl.Producer[*model.Folder] {
|
||||
return ppl.NewProducer(p.produce, ppl.Name("load folders with playlists from db"))
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) produce(put func(entry *model.Folder)) error {
|
||||
u, _ := request.UserFrom(p.ctx)
|
||||
if !conf.Server.AutoImportPlaylists || !u.IsAdmin {
|
||||
log.Warn(p.ctx, "Playlists will not be imported, as there are no admin users yet, "+
|
||||
"Please create an admin user first, and then update the playlists for them to be imported")
|
||||
return nil
|
||||
}
|
||||
|
||||
count := 0
|
||||
cursor, err := p.ds.Folder(p.ctx).GetTouchedWithPlaylists()
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading touched folders: %w", err)
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Checking playlists that may need refresh")
|
||||
for folder, err := range cursor {
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading touched folder: %w", err)
|
||||
}
|
||||
count++
|
||||
put(&folder)
|
||||
}
|
||||
if count == 0 {
|
||||
log.Debug(p.ctx, "Scanner: No playlists need refreshing")
|
||||
} else {
|
||||
log.Debug(p.ctx, "Scanner: Found folders with playlists that may need refreshing", "count", count)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) stages() []ppl.Stage[*model.Folder] {
|
||||
return []ppl.Stage[*model.Folder]{
|
||||
ppl.NewStage(p.processPlaylistsInFolder, ppl.Name("process playlists in folder"), ppl.Concurrency(3)),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) processPlaylistsInFolder(folder *model.Folder) (*model.Folder, error) {
|
||||
files, err := os.ReadDir(folder.AbsolutePath())
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error reading files", "folder", folder, err)
|
||||
p.scanState.sendWarning(err.Error())
|
||||
return folder, nil
|
||||
}
|
||||
for _, f := range files {
|
||||
started := time.Now()
|
||||
if strings.HasPrefix(f.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
if !model.IsValidPlaylist(f.Name()) {
|
||||
continue
|
||||
}
|
||||
// BFR: Check if playlist needs to be refreshed (timestamp, sync flag, etc)
|
||||
pls, err := p.pls.ImportFile(p.ctx, folder, f.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pls.IsSmartPlaylist() {
|
||||
log.Debug("Scanner: Imported smart playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "elapsed", time.Since(started))
|
||||
} else {
|
||||
log.Debug("Scanner: Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started))
|
||||
}
|
||||
p.cw.PreCache(pls.CoverArtID())
|
||||
p.refreshed.Add(1)
|
||||
}
|
||||
return folder, nil
|
||||
}
|
||||
|
||||
func (p *phasePlaylists) finalize(err error) error {
|
||||
refreshed := p.refreshed.Load()
|
||||
logF := log.Info
|
||||
if refreshed == 0 {
|
||||
logF = log.Debug
|
||||
} else {
|
||||
p.scanState.changesDetected.Store(true)
|
||||
}
|
||||
logF(p.ctx, "Scanner: Finished refreshing playlists", "refreshed", refreshed, err)
|
||||
return err
|
||||
}
|
||||
|
||||
var _ phase[*model.Folder] = (*phasePlaylists)(nil)
|
||||
164
scanner/phase_4_playlists_test.go
Normal file
164
scanner/phase_4_playlists_test.go
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
var _ = Describe("phasePlaylists", func() {
|
||||
var (
|
||||
phase *phasePlaylists
|
||||
ctx context.Context
|
||||
state *scanState
|
||||
folderRepo *mockFolderRepository
|
||||
ds *tests.MockDataStore
|
||||
pls *mockPlaylists
|
||||
cw artwork.CacheWarmer
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.AutoImportPlaylists = true
|
||||
ctx = context.Background()
|
||||
ctx = request.WithUser(ctx, model.User{ID: "123", IsAdmin: true})
|
||||
folderRepo = &mockFolderRepository{}
|
||||
ds = &tests.MockDataStore{
|
||||
MockedFolder: folderRepo,
|
||||
}
|
||||
pls = &mockPlaylists{}
|
||||
cw = artwork.NoopCacheWarmer()
|
||||
state = &scanState{}
|
||||
phase = createPhasePlaylists(ctx, state, ds, pls, cw)
|
||||
})
|
||||
|
||||
Describe("description", func() {
|
||||
It("returns the correct description", func() {
|
||||
Expect(phase.description()).To(Equal("Import/update playlists"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("producer", func() {
|
||||
It("produces folders with playlists", func() {
|
||||
folderRepo.SetData(map[*model.Folder]error{
|
||||
{Path: "/path/to/folder1"}: nil,
|
||||
{Path: "/path/to/folder2"}: nil,
|
||||
})
|
||||
|
||||
var produced []*model.Folder
|
||||
err := phase.produce(func(folder *model.Folder) {
|
||||
produced = append(produced, folder)
|
||||
})
|
||||
|
||||
sort.Slice(produced, func(i, j int) bool {
|
||||
return produced[i].Path < produced[j].Path
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(produced).To(HaveLen(2))
|
||||
Expect(produced[0].Path).To(Equal("/path/to/folder1"))
|
||||
Expect(produced[1].Path).To(Equal("/path/to/folder2"))
|
||||
})
|
||||
|
||||
It("returns an error if there is an error loading folders", func() {
|
||||
folderRepo.SetData(map[*model.Folder]error{
|
||||
nil: errors.New("error loading folders"),
|
||||
})
|
||||
|
||||
called := false
|
||||
err := phase.produce(func(folder *model.Folder) { called = true })
|
||||
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(called).To(BeFalse())
|
||||
Expect(err).To(MatchError(ContainSubstring("error loading folders")))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("processPlaylistsInFolder", func() {
|
||||
It("processes playlists in a folder", func() {
|
||||
libPath := GinkgoT().TempDir()
|
||||
folder := &model.Folder{LibraryPath: libPath, Path: "path/to", Name: "folder"}
|
||||
_ = os.MkdirAll(folder.AbsolutePath(), 0755)
|
||||
|
||||
file1 := filepath.Join(folder.AbsolutePath(), "playlist1.m3u")
|
||||
file2 := filepath.Join(folder.AbsolutePath(), "playlist2.m3u")
|
||||
_ = os.WriteFile(file1, []byte{}, 0600)
|
||||
_ = os.WriteFile(file2, []byte{}, 0600)
|
||||
|
||||
pls.On("ImportFile", mock.Anything, folder, "playlist1.m3u").
|
||||
Return(&model.Playlist{}, nil)
|
||||
pls.On("ImportFile", mock.Anything, folder, "playlist2.m3u").
|
||||
Return(&model.Playlist{}, nil)
|
||||
|
||||
_, err := phase.processPlaylistsInFolder(folder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(pls.Calls).To(HaveLen(2))
|
||||
Expect(pls.Calls[0].Arguments[2]).To(Equal("playlist1.m3u"))
|
||||
Expect(pls.Calls[1].Arguments[2]).To(Equal("playlist2.m3u"))
|
||||
Expect(phase.refreshed.Load()).To(Equal(uint32(2)))
|
||||
})
|
||||
|
||||
It("reports an error if there is an error reading files", func() {
|
||||
progress := make(chan *ProgressInfo)
|
||||
state.progress = progress
|
||||
folder := &model.Folder{Path: "/invalid/path"}
|
||||
go func() {
|
||||
_, err := phase.processPlaylistsInFolder(folder)
|
||||
// I/O errors are ignored
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}()
|
||||
|
||||
// But are reported
|
||||
info := &ProgressInfo{}
|
||||
Eventually(progress).Should(Receive(&info))
|
||||
Expect(info.Warning).To(ContainSubstring("no such file or directory"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
type mockPlaylists struct {
|
||||
mock.Mock
|
||||
core.Playlists
|
||||
}
|
||||
|
||||
func (p *mockPlaylists) ImportFile(ctx context.Context, folder *model.Folder, filename string) (*model.Playlist, error) {
|
||||
args := p.Called(ctx, folder, filename)
|
||||
return args.Get(0).(*model.Playlist), args.Error(1)
|
||||
}
|
||||
|
||||
type mockFolderRepository struct {
|
||||
model.FolderRepository
|
||||
data map[*model.Folder]error
|
||||
}
|
||||
|
||||
func (f *mockFolderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error) {
|
||||
return func(yield func(model.Folder, error) bool) {
|
||||
for folder, err := range f.data {
|
||||
if err != nil {
|
||||
if !yield(model.Folder{}, err) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !yield(*folder, err) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *mockFolderRepository) SetData(m map[*model.Folder]error) {
|
||||
f.data = m
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mattn/go-zglob"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type playlistImporter struct {
|
||||
ds model.DataStore
|
||||
pls core.Playlists
|
||||
cacheWarmer artwork.CacheWarmer
|
||||
rootFolder string
|
||||
}
|
||||
|
||||
func newPlaylistImporter(ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer, rootFolder string) *playlistImporter {
|
||||
return &playlistImporter{ds: ds, pls: playlists, cacheWarmer: cacheWarmer, rootFolder: rootFolder}
|
||||
}
|
||||
|
||||
func (s *playlistImporter) processPlaylists(ctx context.Context, dir string) int64 {
|
||||
if !s.inPlaylistsPath(dir) {
|
||||
return 0
|
||||
}
|
||||
var count int64
|
||||
files, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error reading files", "dir", dir, err)
|
||||
return count
|
||||
}
|
||||
for _, f := range files {
|
||||
started := time.Now()
|
||||
if strings.HasPrefix(f.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
if !model.IsValidPlaylist(f.Name()) {
|
||||
continue
|
||||
}
|
||||
pls, err := s.pls.ImportFile(ctx, dir, f.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if pls.IsSmartPlaylist() {
|
||||
log.Debug("Imported smart playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "elapsed", time.Since(started))
|
||||
} else {
|
||||
log.Debug("Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started))
|
||||
}
|
||||
s.cacheWarmer.PreCache(pls.CoverArtID())
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (s *playlistImporter) inPlaylistsPath(dir string) bool {
|
||||
rel, _ := filepath.Rel(s.rootFolder, dir)
|
||||
for _, path := range strings.Split(conf.Server.PlaylistsPath, string(filepath.ListSeparator)) {
|
||||
if match, _ := zglob.Match(path, rel); match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("playlistImporter", func() {
|
||||
var ds model.DataStore
|
||||
var ps *playlistImporter
|
||||
var pls core.Playlists
|
||||
var cw artwork.CacheWarmer
|
||||
ctx := context.Background()
|
||||
|
||||
BeforeEach(func() {
|
||||
ds = &tests.MockDataStore{
|
||||
MockedMediaFile: &mockedMediaFile{},
|
||||
MockedPlaylist: &mockedPlaylist{},
|
||||
}
|
||||
pls = core.NewPlaylists(ds)
|
||||
|
||||
cw = &noopCacheWarmer{}
|
||||
})
|
||||
|
||||
Describe("processPlaylists", func() {
|
||||
Context("Default PlaylistsPath", func() {
|
||||
BeforeEach(func() {
|
||||
conf.Server.PlaylistsPath = consts.DefaultPlaylistsPath
|
||||
})
|
||||
It("finds and import playlists at the top level", func() {
|
||||
ps = newPlaylistImporter(ds, pls, cw, "tests/fixtures/playlists/subfolder1")
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1)))
|
||||
})
|
||||
|
||||
It("finds and import playlists at any subfolder level", func() {
|
||||
ps = newPlaylistImporter(ds, pls, cw, "tests")
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1)))
|
||||
})
|
||||
})
|
||||
|
||||
It("ignores playlists not in the PlaylistsPath", func() {
|
||||
conf.Server.PlaylistsPath = "subfolder1"
|
||||
ps = newPlaylistImporter(ds, pls, cw, "tests/fixtures/playlists")
|
||||
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(1)))
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder2")).To(Equal(int64(0)))
|
||||
})
|
||||
|
||||
It("only imports playlists from the root of MusicFolder if PlaylistsPath is '.'", func() {
|
||||
conf.Server.PlaylistsPath = "."
|
||||
ps = newPlaylistImporter(ds, pls, cw, "tests/fixtures/playlists")
|
||||
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists")).To(Equal(int64(6)))
|
||||
Expect(ps.processPlaylists(ctx, "tests/fixtures/playlists/subfolder1")).To(Equal(int64(0)))
|
||||
})
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
type mockedMediaFile struct {
|
||||
model.MediaFileRepository
|
||||
}
|
||||
|
||||
func (r *mockedMediaFile) FindByPaths(paths []string) (model.MediaFiles, error) {
|
||||
var mfs model.MediaFiles
|
||||
for i, path := range paths {
|
||||
mf := model.MediaFile{
|
||||
ID: strconv.Itoa(i),
|
||||
Path: path,
|
||||
}
|
||||
mfs = append(mfs, mf)
|
||||
}
|
||||
return mfs, nil
|
||||
}
|
||||
|
||||
type mockedPlaylist struct {
|
||||
model.PlaylistRepository
|
||||
}
|
||||
|
||||
func (r *mockedPlaylist) FindByPath(_ string) (*model.Playlist, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (r *mockedPlaylist) Put(_ *model.Playlist) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type noopCacheWarmer struct{}
|
||||
|
||||
func (a *noopCacheWarmer) PreCache(_ model.ArtworkID) {}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
// refresher is responsible for rolling up mediafiles attributes into albums attributes,
|
||||
// and albums attributes into artists attributes. This is done by accumulating all album and artist IDs
|
||||
// found during scan, and "refreshing" the albums and artists when flush is called.
|
||||
//
|
||||
// The actual mappings happen in MediaFiles.ToAlbum() and Albums.ToAlbumArtist()
|
||||
type refresher struct {
|
||||
ds model.DataStore
|
||||
lib model.Library
|
||||
album map[string]struct{}
|
||||
artist map[string]struct{}
|
||||
dirMap dirMap
|
||||
cacheWarmer artwork.CacheWarmer
|
||||
}
|
||||
|
||||
func newRefresher(ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, dirMap dirMap) *refresher {
|
||||
return &refresher{
|
||||
ds: ds,
|
||||
lib: lib,
|
||||
album: map[string]struct{}{},
|
||||
artist: map[string]struct{}{},
|
||||
dirMap: dirMap,
|
||||
cacheWarmer: cw,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *refresher) accumulate(mf model.MediaFile) {
|
||||
if mf.AlbumID != "" {
|
||||
r.album[mf.AlbumID] = struct{}{}
|
||||
}
|
||||
if mf.AlbumArtistID != "" {
|
||||
r.artist[mf.AlbumArtistID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *refresher) flush(ctx context.Context) error {
|
||||
err := r.flushMap(ctx, r.album, "album", r.refreshAlbums)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.album = map[string]struct{}{}
|
||||
err = r.flushMap(ctx, r.artist, "artist", r.refreshArtists)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.artist = map[string]struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
type refreshCallbackFunc = func(ctx context.Context, ids ...string) error
|
||||
|
||||
func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for chunk := range slice.CollectChunks(maps.Keys(m), 200) {
|
||||
err := refresh(ctx, chunk...)
|
||||
if err != nil {
|
||||
log.Error(ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *refresher) refreshAlbums(ctx context.Context, ids ...string) error {
|
||||
mfs, err := r.ds.MediaFile(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(mfs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
repo := r.ds.Album(ctx)
|
||||
grouped := slice.Group(mfs, func(m model.MediaFile) string { return m.AlbumID })
|
||||
for _, group := range grouped {
|
||||
songs := model.MediaFiles(group)
|
||||
a := songs.ToAlbum()
|
||||
var updatedAt time.Time
|
||||
a.ImageFiles, updatedAt = r.getImageFiles(songs.Dirs())
|
||||
if updatedAt.After(a.UpdatedAt) {
|
||||
a.UpdatedAt = updatedAt
|
||||
}
|
||||
a.LibraryID = r.lib.ID
|
||||
err := repo.Put(&a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.cacheWarmer.PreCache(a.CoverArtID())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *refresher) getImageFiles(dirs []string) (string, time.Time) {
|
||||
var imageFiles []string
|
||||
var updatedAt time.Time
|
||||
for _, dir := range dirs {
|
||||
stats := r.dirMap[dir]
|
||||
for _, img := range stats.Images {
|
||||
imageFiles = append(imageFiles, filepath.Join(dir, img))
|
||||
}
|
||||
if stats.ImagesUpdatedAt.After(updatedAt) {
|
||||
updatedAt = stats.ImagesUpdatedAt
|
||||
}
|
||||
}
|
||||
return strings.Join(imageFiles, consts.Zwsp), updatedAt
|
||||
}
|
||||
|
||||
func (r *refresher) refreshArtists(ctx context.Context, ids ...string) error {
|
||||
albums, err := r.ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_artist_id": ids}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(albums) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
repo := r.ds.Artist(ctx)
|
||||
libRepo := r.ds.Library(ctx)
|
||||
grouped := slice.Group(albums, func(al model.Album) string { return al.AlbumArtistID })
|
||||
for _, group := range grouped {
|
||||
a := model.Albums(group).ToAlbumArtist()
|
||||
|
||||
// Force an external metadata lookup on next access
|
||||
a.ExternalInfoUpdatedAt = &time.Time{}
|
||||
|
||||
// Do not remove old metadata
|
||||
err := repo.Put(&a, "album_count", "genres", "external_info_updated_at", "mbz_artist_id", "name", "order_artist_name", "size", "sort_artist_name", "song_count")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Link the artist to the current library being scanned
|
||||
err = libRepo.AddArtist(r.lib.ID, a.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.cacheWarmer.PreCache(a.CoverArtID())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -2,264 +2,243 @@ package scanner
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/utils/singleton"
|
||||
"golang.org/x/time/rate"
|
||||
"github.com/navidrome/navidrome/utils/chain"
|
||||
)
|
||||
|
||||
type Scanner interface {
|
||||
RescanAll(ctx context.Context, fullRescan bool) error
|
||||
Status(library string) (*StatusInfo, error)
|
||||
type scannerImpl struct {
|
||||
ds model.DataStore
|
||||
cw artwork.CacheWarmer
|
||||
pls core.Playlists
|
||||
metrics metrics.Metrics
|
||||
}
|
||||
|
||||
type StatusInfo struct {
|
||||
Library string
|
||||
Scanning bool
|
||||
LastScan time.Time
|
||||
Count uint32
|
||||
FolderCount uint32
|
||||
// scanState holds the state of an in-progress scan, to be passed to the various phases
|
||||
type scanState struct {
|
||||
progress chan<- *ProgressInfo
|
||||
fullScan bool
|
||||
changesDetected atomic.Bool
|
||||
}
|
||||
|
||||
var (
|
||||
ErrAlreadyScanning = errors.New("already scanning")
|
||||
ErrScanError = errors.New("scan error")
|
||||
)
|
||||
|
||||
type FolderScanner interface {
|
||||
// Scan process finds any changes after `lastModifiedSince` and returns the number of changes found
|
||||
Scan(ctx context.Context, lib model.Library, fullRescan bool, progress chan uint32) (int64, error)
|
||||
}
|
||||
|
||||
var isScanning sync.Mutex
|
||||
|
||||
type scanner struct {
|
||||
once sync.Once
|
||||
folders map[string]FolderScanner
|
||||
libs map[string]model.Library
|
||||
status map[string]*scanStatus
|
||||
lock *sync.RWMutex
|
||||
ds model.DataStore
|
||||
pls core.Playlists
|
||||
broker events.Broker
|
||||
cacheWarmer artwork.CacheWarmer
|
||||
metrics metrics.Metrics
|
||||
}
|
||||
|
||||
type scanStatus struct {
|
||||
active bool
|
||||
fileCount uint32
|
||||
folderCount uint32
|
||||
lastUpdate time.Time
|
||||
}
|
||||
|
||||
func GetInstance(ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer, broker events.Broker, metrics metrics.Metrics) Scanner {
|
||||
return singleton.GetInstance(func() *scanner {
|
||||
s := &scanner{
|
||||
ds: ds,
|
||||
pls: playlists,
|
||||
broker: broker,
|
||||
folders: map[string]FolderScanner{},
|
||||
libs: map[string]model.Library{},
|
||||
status: map[string]*scanStatus{},
|
||||
lock: &sync.RWMutex{},
|
||||
cacheWarmer: cacheWarmer,
|
||||
metrics: metrics,
|
||||
}
|
||||
s.loadFolders()
|
||||
return s
|
||||
})
|
||||
}
|
||||
|
||||
func (s *scanner) rescan(ctx context.Context, library string, fullRescan bool) error {
|
||||
folderScanner := s.folders[library]
|
||||
start := time.Now()
|
||||
|
||||
lib, ok := s.libs[library]
|
||||
if !ok {
|
||||
log.Error(ctx, "Folder not a valid library path", "folder", library)
|
||||
return fmt.Errorf("folder %s not a valid library path", library)
|
||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
if s.progress != nil {
|
||||
s.progress <- info
|
||||
}
|
||||
}
|
||||
|
||||
s.setStatusStart(library)
|
||||
defer s.setStatusEnd(library, start)
|
||||
func (s *scanState) sendWarning(msg string) {
|
||||
s.sendProgress(&ProgressInfo{Warning: msg})
|
||||
}
|
||||
|
||||
if fullRescan {
|
||||
log.Debug("Scanning folder (full scan)", "folder", library)
|
||||
} else {
|
||||
log.Debug("Scanning folder", "folder", library, "lastScan", lib.LastScanAt)
|
||||
}
|
||||
func (s *scanState) sendError(err error) {
|
||||
s.sendProgress(&ProgressInfo{Error: err.Error()})
|
||||
}
|
||||
|
||||
progress, cancel := s.startProgressTracker(library)
|
||||
defer cancel()
|
||||
|
||||
changeCount, err := folderScanner.Scan(ctx, lib, fullRescan, progress)
|
||||
func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
|
||||
state := scanState{progress: progress, fullScan: fullScan}
|
||||
libs, err := s.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
log.Error("Error scanning Library", "folder", library, err)
|
||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
if changeCount > 0 {
|
||||
log.Debug(ctx, "Detected changes in the music folder. Sending refresh event",
|
||||
"folder", library, "changeCount", changeCount)
|
||||
// Don't use real context, forcing a refresh in all open windows, including the one that triggered the scan
|
||||
s.broker.SendMessage(context.Background(), &events.RefreshResource{})
|
||||
}
|
||||
startTime := time.Now()
|
||||
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs))
|
||||
|
||||
s.updateLastModifiedSince(ctx, library, start)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *scanner) startProgressTracker(library string) (chan uint32, context.CancelFunc) {
|
||||
// Must be a new context (not the one passed to the scan method) to allow broadcasting the scan status to all clients
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
progress := make(chan uint32, 1000)
|
||||
limiter := rate.Sometimes{Interval: conf.Server.DevActivityPanelUpdateRate}
|
||||
go func() {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{Scanning: true, Count: 0, FolderCount: 0})
|
||||
defer func() {
|
||||
if status, ok := s.getStatus(library); ok {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{
|
||||
Scanning: false,
|
||||
Count: int64(status.fileCount),
|
||||
FolderCount: int64(status.folderCount),
|
||||
})
|
||||
}
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case count := <-progress:
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
totalFolders, totalFiles := s.incStatusCounter(library, count)
|
||||
limiter.Do(func() {
|
||||
s.broker.SendMessage(ctx, &events.ScanStatus{
|
||||
Scanning: true,
|
||||
Count: int64(totalFiles),
|
||||
FolderCount: int64(totalFolders),
|
||||
})
|
||||
})
|
||||
// if there was a full scan in progress, force a full scan
|
||||
if !state.fullScan {
|
||||
for _, lib := range libs {
|
||||
if lib.FullScanInProgress {
|
||||
log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name)
|
||||
state.fullScan = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return progress, cancel
|
||||
}
|
||||
|
||||
func (s *scanner) getStatus(folder string) (scanStatus, bool) {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
status, ok := s.status[folder]
|
||||
return *status, ok
|
||||
}
|
||||
|
||||
func (s *scanner) incStatusCounter(folder string, numFiles uint32) (totalFolders uint32, totalFiles uint32) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.fileCount += numFiles
|
||||
status.folderCount++
|
||||
totalFolders = status.folderCount
|
||||
totalFiles = status.fileCount
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *scanner) setStatusStart(folder string) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.active = true
|
||||
status.fileCount = 0
|
||||
status.folderCount = 0
|
||||
}
|
||||
}
|
||||
err = chain.RunSequentially(
|
||||
// Phase 1: Scan all libraries and import new/updated files
|
||||
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)),
|
||||
|
||||
func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if status, ok := s.status[folder]; ok {
|
||||
status.active = false
|
||||
status.lastUpdate = lastUpdate
|
||||
}
|
||||
}
|
||||
// Phase 2: Process missing files, checking for moves
|
||||
runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)),
|
||||
|
||||
func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
s.once.Do(s.loadFolders)
|
||||
// Phases 3 and 4 can be run in parallel
|
||||
chain.RunParallel(
|
||||
// Phase 3: Refresh all new/changed albums and update artists
|
||||
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)),
|
||||
|
||||
if !isScanning.TryLock() {
|
||||
log.Debug(ctx, "Scanner already running, ignoring request for rescan.")
|
||||
return ErrAlreadyScanning
|
||||
}
|
||||
defer isScanning.Unlock()
|
||||
// Phase 4: Import/update playlists
|
||||
runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)),
|
||||
),
|
||||
|
||||
var hasError bool
|
||||
for folder := range s.folders {
|
||||
err := s.rescan(ctx, folder, fullRescan)
|
||||
hasError = hasError || err != nil
|
||||
}
|
||||
if hasError {
|
||||
log.Error(ctx, "Errors while scanning media. Please check the logs")
|
||||
// Final Steps (cannot be parallelized):
|
||||
|
||||
// Run GC if there were any changes (Remove dangling tracks, empty albums and artists, and orphan annotations)
|
||||
s.runGC(ctx, &state),
|
||||
|
||||
// Refresh artist and tags stats
|
||||
s.runRefreshStats(ctx, &state),
|
||||
|
||||
// Update last_scan_completed_at for all libraries
|
||||
s.runUpdateLibraries(ctx, libs),
|
||||
|
||||
// Optimize DB
|
||||
s.runOptimize(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err)
|
||||
state.sendError(err)
|
||||
s.metrics.WriteAfterScanMetrics(ctx, false)
|
||||
return ErrScanError
|
||||
return
|
||||
}
|
||||
s.metrics.WriteAfterScanMetrics(ctx, true)
|
||||
return nil
|
||||
|
||||
if state.changesDetected.Load() {
|
||||
state.sendProgress(&ProgressInfo{ChangesDetected: true})
|
||||
}
|
||||
|
||||
s.metrics.WriteAfterScanMetrics(ctx, err == nil)
|
||||
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
|
||||
}
|
||||
|
||||
func (s *scanner) Status(library string) (*StatusInfo, error) {
|
||||
s.once.Do(s.loadFolders)
|
||||
status, ok := s.getStatus(library)
|
||||
if !ok {
|
||||
return nil, errors.New("library not found")
|
||||
func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error {
|
||||
return func() error {
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
if state.changesDetected.Load() {
|
||||
start := time.Now()
|
||||
err := tx.GC(ctx)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error running GC", err)
|
||||
return fmt.Errorf("running GC: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: GC completed", "elapsed", time.Since(start))
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: No changes detected, skipping GC")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return &StatusInfo{
|
||||
Library: library,
|
||||
Scanning: status.active,
|
||||
LastScan: status.lastUpdate,
|
||||
Count: status.fileCount,
|
||||
FolderCount: status.folderCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *scanner) updateLastModifiedSince(ctx context.Context, folder string, t time.Time) {
|
||||
lib := s.libs[folder]
|
||||
id := lib.ID
|
||||
if err := s.ds.Library(ctx).UpdateLastScan(id, t); err != nil {
|
||||
log.Error("Error updating DB after scan", err)
|
||||
}
|
||||
lib.LastScanAt = t
|
||||
s.libs[folder] = lib
|
||||
}
|
||||
|
||||
func (s *scanner) loadFolders() {
|
||||
ctx := context.TODO()
|
||||
libs, _ := s.ds.Library(ctx).GetAll()
|
||||
for _, lib := range libs {
|
||||
log.Info("Configuring Media Folder", "name", lib.Name, "path", lib.Path)
|
||||
s.folders[lib.Path] = s.newScanner()
|
||||
s.libs[lib.Path] = lib
|
||||
s.status[lib.Path] = &scanStatus{
|
||||
active: false,
|
||||
fileCount: 0,
|
||||
folderCount: 0,
|
||||
lastUpdate: lib.LastScanAt,
|
||||
func (s *scannerImpl) runRefreshStats(ctx context.Context, state *scanState) func() error {
|
||||
return func() error {
|
||||
if !state.changesDetected.Load() {
|
||||
log.Debug(ctx, "Scanner: No changes detected, skipping refreshing stats")
|
||||
return nil
|
||||
}
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
start := time.Now()
|
||||
stats, err := tx.Artist(ctx).RefreshStats()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error refreshing artists stats", err)
|
||||
return fmt.Errorf("refreshing artists stats: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Refreshed artist stats", "stats", stats, "elapsed", time.Since(start))
|
||||
|
||||
start = time.Now()
|
||||
err = tx.Tag(ctx).UpdateCounts()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating tag counts", err)
|
||||
return fmt.Errorf("updating tag counts: %w", err)
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Updated tag counts", "elapsed", time.Since(start))
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scanner) newScanner() FolderScanner {
|
||||
return NewTagScanner(s.ds, s.pls, s.cacheWarmer)
|
||||
func (s *scannerImpl) runOptimize(ctx context.Context) func() error {
|
||||
return func() error {
|
||||
start := time.Now()
|
||||
db.Optimize(ctx)
|
||||
log.Debug(ctx, "Scanner: Optimized DB", "elapsed", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scannerImpl) runUpdateLibraries(ctx context.Context, libs model.Libraries) func() error {
|
||||
return func() error {
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
for _, lib := range libs {
|
||||
err := tx.Library(ctx).ScanEnd(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating last scan completed", "lib", lib.Name, err)
|
||||
return fmt.Errorf("updating last scan completed: %w", err)
|
||||
}
|
||||
err = tx.Property(ctx).Put(consts.PIDTrackKey, conf.Server.PID.Track)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating track PID conf", err)
|
||||
return fmt.Errorf("updating track PID conf: %w", err)
|
||||
}
|
||||
err = tx.Property(ctx).Put(consts.PIDAlbumKey, conf.Server.PID.Album)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating album PID conf", err)
|
||||
return fmt.Errorf("updating album PID conf: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type phase[T any] interface {
|
||||
producer() ppl.Producer[T]
|
||||
stages() []ppl.Stage[T]
|
||||
finalize(error) error
|
||||
description() string
|
||||
}
|
||||
|
||||
func runPhase[T any](ctx context.Context, phaseNum int, phase phase[T]) func() error {
|
||||
return func() error {
|
||||
log.Debug(ctx, fmt.Sprintf("Scanner: Starting phase %d: %s", phaseNum, phase.description()))
|
||||
start := time.Now()
|
||||
|
||||
producer := phase.producer()
|
||||
stages := phase.stages()
|
||||
|
||||
// Prepend a counter stage to the phase's pipeline
|
||||
counter, countStageFn := countTasks[T]()
|
||||
stages = append([]ppl.Stage[T]{ppl.NewStage(countStageFn, ppl.Name("count tasks"))}, stages...)
|
||||
|
||||
var err error
|
||||
if log.IsGreaterOrEqualTo(log.LevelDebug) {
|
||||
var m *ppl.Metrics
|
||||
m, err = ppl.Measure(producer, stages...)
|
||||
log.Info(ctx, "Scanner: "+m.String(), err)
|
||||
} else {
|
||||
err = ppl.Do(producer, stages...)
|
||||
}
|
||||
|
||||
err = phase.finalize(err)
|
||||
|
||||
if err != nil {
|
||||
log.Error(ctx, fmt.Sprintf("Scanner: Error processing libraries in phase %d", phaseNum), "elapsed", time.Since(start), err)
|
||||
} else {
|
||||
log.Debug(ctx, fmt.Sprintf("Scanner: Finished phase %d", phaseNum), "elapsed", time.Since(start), "totalTasks", counter.Load())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func countTasks[T any]() (*atomic.Int64, func(T) (T, error)) {
|
||||
counter := atomic.Int64{}
|
||||
return &counter, func(in T) (T, error) {
|
||||
counter.Add(1)
|
||||
return in, nil
|
||||
}
|
||||
}
|
||||
|
||||
var _ scanner = (*scannerImpl)(nil)
|
||||
|
|
|
|||
89
scanner/scanner_benchmark_test.go
Normal file
89
scanner/scanner_benchmark_test.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
package scanner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/google/uuid"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/core/storage/storagetest"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func BenchmarkScan(b *testing.B) {
|
||||
// Detect any goroutine leaks in the scanner code under test
|
||||
defer goleak.VerifyNone(b,
|
||||
goleak.IgnoreTopFunction("testing.(*B).run1"),
|
||||
goleak.IgnoreAnyFunction("testing.(*B).doBench"),
|
||||
// Ignore database/sql.(*DB).connectionOpener, as we are not closing the database connection
|
||||
goleak.IgnoreAnyFunction("database/sql.(*DB).connectionOpener"),
|
||||
)
|
||||
|
||||
tmpDir := os.TempDir()
|
||||
conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL")
|
||||
db.Init(context.Background())
|
||||
|
||||
ds := persistence.New(db.Db())
|
||||
conf.Server.DevExternalScanner = false
|
||||
s := scanner.New(context.Background(), ds, artwork.NoopCacheWarmer(), events.NoopBroker(),
|
||||
core.NewPlaylists(ds), metrics.NewNoopInstance())
|
||||
|
||||
fs := storagetest.FakeFS{}
|
||||
storagetest.Register("fake", &fs)
|
||||
var beatlesMBID = uuid.NewString()
|
||||
beatles := _t{
|
||||
"artist": "The Beatles",
|
||||
"artistsort": "Beatles, The",
|
||||
"musicbrainz_artistid": beatlesMBID,
|
||||
"albumartist": "The Beatles",
|
||||
"albumartistsort": "Beatles The",
|
||||
"musicbrainz_albumartistid": beatlesMBID,
|
||||
}
|
||||
revolver := template(beatles, _t{"album": "Revolver", "year": 1966, "composer": "Lennon/McCartney"})
|
||||
help := template(beatles, _t{"album": "Help!", "year": 1965, "composer": "Lennon/McCartney"})
|
||||
fs.SetFiles(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(track(2, "Eleanor Rigby")),
|
||||
"The Beatles/Revolver/03 - I'm Only Sleeping.mp3": revolver(track(3, "I'm Only Sleeping")),
|
||||
"The Beatles/Revolver/04 - Love You To.mp3": revolver(track(4, "Love You To")),
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(track(1, "Help!")),
|
||||
"The Beatles/Help!/02 - The Night Before.mp3": help(track(2, "The Night Before")),
|
||||
"The Beatles/Help!/03 - You've Got to Hide Your Love Away.mp3": help(track(3, "You've Got to Hide Your Love Away")),
|
||||
})
|
||||
|
||||
lib := model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"}
|
||||
err := ds.Library(context.Background()).Put(&lib)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
var m1, m2 runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&m1)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := s.ScanAll(context.Background(), true)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
runtime.ReadMemStats(&m2)
|
||||
fmt.Println("total:", humanize.Bytes(m2.TotalAlloc-m1.TotalAlloc))
|
||||
fmt.Println("mallocs:", humanize.Comma(int64(m2.Mallocs-m1.Mallocs)))
|
||||
}
|
||||
98
scanner/scanner_internal_test.go
Normal file
98
scanner/scanner_internal_test.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
// nolint unused
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
|
||||
ppl "github.com/google/go-pipeline/pkg/pipeline"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type mockPhase struct {
|
||||
num int
|
||||
produceFunc func() ppl.Producer[int]
|
||||
stagesFunc func() []ppl.Stage[int]
|
||||
finalizeFunc func(error) error
|
||||
descriptionFn func() string
|
||||
}
|
||||
|
||||
func (m *mockPhase) producer() ppl.Producer[int] {
|
||||
return m.produceFunc()
|
||||
}
|
||||
|
||||
func (m *mockPhase) stages() []ppl.Stage[int] {
|
||||
return m.stagesFunc()
|
||||
}
|
||||
|
||||
func (m *mockPhase) finalize(err error) error {
|
||||
return m.finalizeFunc(err)
|
||||
}
|
||||
|
||||
func (m *mockPhase) description() string {
|
||||
return m.descriptionFn()
|
||||
}
|
||||
|
||||
var _ = Describe("runPhase", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
phaseNum int
|
||||
phase *mockPhase
|
||||
sum atomic.Int32
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
phaseNum = 1
|
||||
phase = &mockPhase{
|
||||
num: 3,
|
||||
produceFunc: func() ppl.Producer[int] {
|
||||
return ppl.NewProducer(func(put func(int)) error {
|
||||
for i := 1; i <= phase.num; i++ {
|
||||
put(i)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
stagesFunc: func() []ppl.Stage[int] {
|
||||
return []ppl.Stage[int]{ppl.NewStage(func(i int) (int, error) {
|
||||
sum.Add(int32(i))
|
||||
return i, nil
|
||||
})}
|
||||
},
|
||||
finalizeFunc: func(err error) error {
|
||||
return err
|
||||
},
|
||||
descriptionFn: func() string {
|
||||
return "Mock Phase"
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
It("should run the phase successfully", func() {
|
||||
err := runPhase(ctx, phaseNum, phase)()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(sum.Load()).To(Equal(int32(1 * 2 * 3)))
|
||||
})
|
||||
|
||||
It("should log an error if the phase fails", func() {
|
||||
phase.finalizeFunc = func(err error) error {
|
||||
return errors.New("finalize error")
|
||||
}
|
||||
err := runPhase(ctx, phaseNum, phase)()
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("finalize error"))
|
||||
})
|
||||
|
||||
It("should count the tasks", func() {
|
||||
counter, countStageFn := countTasks[int]()
|
||||
phase.stagesFunc = func() []ppl.Stage[int] {
|
||||
return []ppl.Stage[int]{ppl.NewStage(countStageFn, ppl.Name("count tasks"))}
|
||||
}
|
||||
err := runPhase(ctx, phaseNum, phase)()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(counter.Load()).To(Equal(int64(3)))
|
||||
})
|
||||
})
|
||||
|
|
@ -1,20 +1,25 @@
|
|||
package scanner
|
||||
package scanner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func TestScanner(t *testing.T) {
|
||||
// Detect any goroutine leaks in the scanner code under test
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"),
|
||||
)
|
||||
|
||||
tests.Init(t, true)
|
||||
conf.Server.DbPath = "file::memory:?cache=shared"
|
||||
defer db.Init()()
|
||||
defer db.Close(context.Background())
|
||||
log.SetLevel(log.LevelFatal)
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Scanner Suite")
|
||||
|
|
|
|||
530
scanner/scanner_test.go
Normal file
530
scanner/scanner_test.go
Normal file
|
|
@ -0,0 +1,530 @@
|
|||
package scanner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/google/uuid"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/core/storage/storagetest"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// Easy aliases for the storagetest package
|
||||
type _t = map[string]any
|
||||
|
||||
var template = storagetest.Template
|
||||
var track = storagetest.Track
|
||||
|
||||
var _ = Describe("Scanner", Ordered, func() {
|
||||
var ctx context.Context
|
||||
var lib model.Library
|
||||
var ds *tests.MockDataStore
|
||||
var mfRepo *mockMediaFileRepo
|
||||
var s scanner.Scanner
|
||||
|
||||
createFS := func(files fstest.MapFS) storagetest.FakeFS {
|
||||
fs := storagetest.FakeFS{}
|
||||
fs.SetFiles(files)
|
||||
storagetest.Register("fake", &fs)
|
||||
return fs
|
||||
}
|
||||
|
||||
BeforeAll(func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL")
|
||||
log.Warn("Using DB at " + conf.Server.DbPath)
|
||||
//conf.Server.DbPath = ":memory:"
|
||||
})
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
db.Init(ctx)
|
||||
DeferCleanup(func() {
|
||||
Expect(tests.ClearDB()).To(Succeed())
|
||||
})
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.DevExternalScanner = false
|
||||
|
||||
ds = &tests.MockDataStore{RealDS: persistence.New(db.Db())}
|
||||
mfRepo = &mockMediaFileRepo{
|
||||
MediaFileRepository: ds.RealDS.MediaFile(ctx),
|
||||
}
|
||||
ds.MockedMediaFile = mfRepo
|
||||
|
||||
s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(),
|
||||
core.NewPlaylists(ds), metrics.NewNoopInstance())
|
||||
|
||||
lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"}
|
||||
Expect(ds.Library(ctx).Put(&lib)).To(Succeed())
|
||||
})
|
||||
|
||||
runScanner := func(ctx context.Context, fullScan bool) error {
|
||||
_, err := s.ScanAll(ctx, fullScan)
|
||||
return err
|
||||
}
|
||||
|
||||
Context("Simple library, 'artis/album/track - title.mp3'", func() {
|
||||
var help, revolver func(...map[string]any) *fstest.MapFile
|
||||
var fsys storagetest.FakeFS
|
||||
BeforeEach(func() {
|
||||
revolver = template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
|
||||
help = template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965})
|
||||
fsys = createFS(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(track(2, "Eleanor Rigby")),
|
||||
"The Beatles/Revolver/03 - I'm Only Sleeping.mp3": revolver(track(3, "I'm Only Sleeping")),
|
||||
"The Beatles/Revolver/04 - Love You To.mp3": revolver(track(4, "Love You To")),
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(track(1, "Help!")),
|
||||
"The Beatles/Help!/02 - The Night Before.mp3": help(track(2, "The Night Before")),
|
||||
"The Beatles/Help!/03 - You've Got to Hide Your Love Away.mp3": help(track(3, "You've Got to Hide Your Love Away")),
|
||||
})
|
||||
})
|
||||
When("it is the first scan", func() {
|
||||
It("should import all folders", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
folders, _ := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}})
|
||||
paths := slice.Map(folders, func(f model.Folder) string { return f.Name })
|
||||
Expect(paths).To(SatisfyAll(
|
||||
HaveLen(4),
|
||||
ContainElements(".", "The Beatles", "Revolver", "Help!"),
|
||||
))
|
||||
})
|
||||
It("should import all mediafiles", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
mfs, _ := ds.MediaFile(ctx).GetAll()
|
||||
paths := slice.Map(mfs, func(f model.MediaFile) string { return f.Title })
|
||||
Expect(paths).To(SatisfyAll(
|
||||
HaveLen(7),
|
||||
ContainElements(
|
||||
"Taxman", "Eleanor Rigby", "I'm Only Sleeping", "Love You To",
|
||||
"Help!", "The Night Before", "You've Got to Hide Your Love Away",
|
||||
),
|
||||
))
|
||||
})
|
||||
It("should import all albums", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
albums, _ := ds.Album(ctx).GetAll(model.QueryOptions{Sort: "name"})
|
||||
Expect(albums).To(HaveLen(2))
|
||||
Expect(albums[0]).To(SatisfyAll(
|
||||
HaveField("Name", Equal("Help!")),
|
||||
HaveField("SongCount", Equal(3)),
|
||||
))
|
||||
Expect(albums[1]).To(SatisfyAll(
|
||||
HaveField("Name", Equal("Revolver")),
|
||||
HaveField("SongCount", Equal(4)),
|
||||
))
|
||||
})
|
||||
})
|
||||
When("a file was changed", func() {
|
||||
It("should update the media_file", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
mf, err := ds.MediaFile(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"title": "Help!"}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf[0].Tags).ToNot(HaveKey("barcode"))
|
||||
|
||||
fsys.UpdateTags("The Beatles/Help!/01 - Help!.mp3", _t{"barcode": "123"})
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
mf, err = ds.MediaFile(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"title": "Help!"}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf[0].Tags).To(HaveKeyWithValue(model.TagName("barcode"), []string{"123"}))
|
||||
})
|
||||
|
||||
It("should update the album", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
albums, err := ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album.name": "Help!"}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(albums).ToNot(BeEmpty())
|
||||
Expect(albums[0].Participants.First(model.RoleProducer).Name).To(BeEmpty())
|
||||
Expect(albums[0].SongCount).To(Equal(3))
|
||||
|
||||
fsys.UpdateTags("The Beatles/Help!/01 - Help!.mp3", _t{"producer": "George Martin"})
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
albums, err = ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album.name": "Help!"}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(albums[0].Participants.First(model.RoleProducer).Name).To(Equal("George Martin"))
|
||||
Expect(albums[0].SongCount).To(Equal(3))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Context("Ignored entries", func() {
|
||||
BeforeEach(func() {
|
||||
revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
|
||||
createFS(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")),
|
||||
"The Beatles/Revolver/._01 - Taxman.mp3": &fstest.MapFile{Data: []byte("garbage data")},
|
||||
})
|
||||
})
|
||||
|
||||
It("should not import the ignored file", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
mfs, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mfs).To(HaveLen(1))
|
||||
for _, mf := range mfs {
|
||||
Expect(mf.Title).To(Equal("Taxman"))
|
||||
Expect(mf.Path).To(Equal("The Beatles/Revolver/01 - Taxman.mp3"))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Context("Same album in two different folders", func() {
|
||||
BeforeEach(func() {
|
||||
revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
|
||||
createFS(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")),
|
||||
"The Beatles/Revolver2/02 - Eleanor Rigby.mp3": revolver(track(2, "Eleanor Rigby")),
|
||||
})
|
||||
})
|
||||
|
||||
It("should import as one album", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
albums, err := ds.Album(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(albums).To(HaveLen(1))
|
||||
|
||||
mfs, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mfs).To(HaveLen(2))
|
||||
for _, mf := range mfs {
|
||||
Expect(mf.AlbumID).To(Equal(albums[0].ID))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Context("Same album, different release dates", func() {
|
||||
BeforeEach(func() {
|
||||
help := template(_t{"albumartist": "The Beatles", "album": "Help!", "releasedate": 1965})
|
||||
help2 := template(_t{"albumartist": "The Beatles", "album": "Help!", "releasedate": 2000})
|
||||
createFS(fstest.MapFS{
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(track(1, "Help!")),
|
||||
"The Beatles/Help! (remaster)/01 - Help!.mp3": help2(track(1, "Help!")),
|
||||
})
|
||||
})
|
||||
|
||||
It("should import as two distinct albums", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
albums, err := ds.Album(ctx).GetAll(model.QueryOptions{Sort: "release_date"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(albums).To(HaveLen(2))
|
||||
Expect(albums[0]).To(SatisfyAll(
|
||||
HaveField("Name", Equal("Help!")),
|
||||
HaveField("ReleaseDate", Equal("1965")),
|
||||
))
|
||||
Expect(albums[1]).To(SatisfyAll(
|
||||
HaveField("Name", Equal("Help!")),
|
||||
HaveField("ReleaseDate", Equal("2000")),
|
||||
))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Library changes'", func() {
|
||||
var help, revolver func(...map[string]any) *fstest.MapFile
|
||||
var fsys storagetest.FakeFS
|
||||
var findByPath func(string) (*model.MediaFile, error)
|
||||
var beatlesMBID = uuid.NewString()
|
||||
|
||||
BeforeEach(func() {
|
||||
By("Having two MP3 albums")
|
||||
beatles := _t{
|
||||
"artist": "The Beatles",
|
||||
"artistsort": "Beatles, The",
|
||||
"musicbrainz_artistid": beatlesMBID,
|
||||
}
|
||||
help = template(beatles, _t{"album": "Help!", "year": 1965})
|
||||
revolver = template(beatles, _t{"album": "Revolver", "year": 1966})
|
||||
fsys = createFS(fstest.MapFS{
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(track(1, "Help!")),
|
||||
"The Beatles/Help!/02 - The Night Before.mp3": help(track(2, "The Night Before")),
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(track(2, "Eleanor Rigby")),
|
||||
})
|
||||
|
||||
By("Doing a full scan")
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
Expect(ds.MediaFile(ctx).CountAll()).To(Equal(int64(4)))
|
||||
findByPath = createFindByPath(ctx, ds)
|
||||
})
|
||||
|
||||
It("adds new files to the library", func() {
|
||||
fsys.Add("The Beatles/Revolver/03 - I'm Only Sleeping.mp3", revolver(track(3, "I'm Only Sleeping")))
|
||||
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
Expect(ds.MediaFile(ctx).CountAll()).To(Equal(int64(5)))
|
||||
mf, err := findByPath("The Beatles/Revolver/03 - I'm Only Sleeping.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Title).To(Equal("I'm Only Sleeping"))
|
||||
})
|
||||
|
||||
It("updates tags of a file in the library", func() {
|
||||
fsys.UpdateTags("The Beatles/Revolver/02 - Eleanor Rigby.mp3", _t{"title": "Eleanor Rigby (remix)"})
|
||||
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
Expect(ds.MediaFile(ctx).CountAll()).To(Equal(int64(4)))
|
||||
mf, _ := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(mf.Title).To(Equal("Eleanor Rigby (remix)"))
|
||||
})
|
||||
|
||||
It("upgrades file with same format in the library", func() {
|
||||
fsys.Add("The Beatles/Revolver/01 - Taxman.mp3", revolver(track(1, "Taxman", _t{"bitrate": 640})))
|
||||
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
Expect(ds.MediaFile(ctx).CountAll()).To(Equal(int64(4)))
|
||||
mf, _ := findByPath("The Beatles/Revolver/01 - Taxman.mp3")
|
||||
Expect(mf.BitRate).To(Equal(640))
|
||||
})
|
||||
|
||||
It("detects a file was removed from the library", func() {
|
||||
By("Removing a file")
|
||||
fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
|
||||
By("Rescanning the library")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the file is marked as missing")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(3)))
|
||||
mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
})
|
||||
|
||||
It("detects a file was moved to a different folder", func() {
|
||||
By("Storing the original ID")
|
||||
original, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
originalId := original.ID
|
||||
|
||||
By("Moving the file to a different folder")
|
||||
fsys.Move("The Beatles/Revolver/02 - Eleanor Rigby.mp3", "The Beatles/Help!/02 - Eleanor Rigby.mp3")
|
||||
|
||||
By("Rescanning the library")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the old file is not in the library")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(4)))
|
||||
_, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
|
||||
By("Checking the new file is in the library")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": true},
|
||||
})).To(BeZero())
|
||||
mf, err := findByPath("The Beatles/Help!/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Title).To(Equal("Eleanor Rigby"))
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
|
||||
By("Checking the new file has the same ID as the original")
|
||||
Expect(mf.ID).To(Equal(originalId))
|
||||
})
|
||||
|
||||
It("detects a move after a scan is interrupted by an error", func() {
|
||||
By("Storing the original ID")
|
||||
By("Moving the file to a different folder")
|
||||
fsys.Move("The Beatles/Revolver/01 - Taxman.mp3", "The Beatles/Help!/01 - Taxman.mp3")
|
||||
|
||||
By("Interrupting the scan with an error before the move is processed")
|
||||
mfRepo.GetMissingAndMatchingError = errors.New("I/O read error")
|
||||
Expect(runScanner(ctx, false)).To(MatchError(ContainSubstring("I/O read error")))
|
||||
|
||||
By("Checking the both instances of the file are in the lib")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"title": "Taxman"},
|
||||
})).To(Equal(int64(2)))
|
||||
|
||||
By("Rescanning the library without error")
|
||||
mfRepo.GetMissingAndMatchingError = nil
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the old file is not in the library")
|
||||
mfs, err := ds.MediaFile(ctx).GetAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"title": "Taxman"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mfs).To(HaveLen(1))
|
||||
Expect(mfs[0].Path).To(Equal("The Beatles/Help!/01 - Taxman.mp3"))
|
||||
})
|
||||
|
||||
It("detects file format upgrades", func() {
|
||||
By("Storing the original ID")
|
||||
original, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
originalId := original.ID
|
||||
|
||||
By("Replacing the file with a different format")
|
||||
fsys.Move("The Beatles/Revolver/02 - Eleanor Rigby.mp3", "The Beatles/Revolver/02 - Eleanor Rigby.flac")
|
||||
|
||||
By("Rescanning the library")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the old file is not in the library")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": true},
|
||||
})).To(BeZero())
|
||||
_, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
|
||||
By("Checking the new file is in the library")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(4)))
|
||||
mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.flac")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Title).To(Equal("Eleanor Rigby"))
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
|
||||
By("Checking the new file has the same ID as the original")
|
||||
Expect(mf.ID).To(Equal(originalId))
|
||||
})
|
||||
|
||||
It("detects old missing tracks being added back", func() {
|
||||
By("Removing a file")
|
||||
origFile := fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
|
||||
By("Rescanning the library")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the file is marked as missing")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(3)))
|
||||
mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
|
||||
By("Adding the file back")
|
||||
fsys.Add("The Beatles/Revolver/02 - Eleanor Rigby.mp3", origFile)
|
||||
|
||||
By("Rescanning the library again")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the file is not marked as missing")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(4)))
|
||||
mf, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
|
||||
By("Removing it again")
|
||||
fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
|
||||
By("Rescanning the library again")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the file is marked as missing")
|
||||
mf, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
|
||||
By("Adding the file back in a different folder")
|
||||
fsys.Add("The Beatles/Help!/02 - Eleanor Rigby.mp3", origFile)
|
||||
|
||||
By("Rescanning the library once more")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Checking the file was found in the new folder")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(4)))
|
||||
mf, err = findByPath("The Beatles/Help!/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
})
|
||||
|
||||
It("does not override artist fields when importing an undertagged file", func() {
|
||||
By("Making sure artist in the DB contains MBID and sort name")
|
||||
aa, err := ds.Artist(ctx).GetAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"name": "The Beatles"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(aa).To(HaveLen(1))
|
||||
Expect(aa[0].Name).To(Equal("The Beatles"))
|
||||
Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID))
|
||||
Expect(aa[0].SortArtistName).To(Equal("Beatles, The"))
|
||||
|
||||
By("Adding a new undertagged file (no MBID or sort name)")
|
||||
newTrack := revolver(track(4, "Love You Too",
|
||||
_t{"artist": "The Beatles", "musicbrainz_artistid": "", "artistsort": ""}),
|
||||
)
|
||||
fsys.Add("The Beatles/Revolver/04 - Love You Too.mp3", newTrack)
|
||||
|
||||
By("Doing a partial scan")
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Asserting MediaFile have the artist name, but not the MBID or sort name")
|
||||
mf, err := findByPath("The Beatles/Revolver/04 - Love You Too.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Title).To(Equal("Love You Too"))
|
||||
Expect(mf.AlbumArtist).To(Equal("The Beatles"))
|
||||
Expect(mf.MbzAlbumArtistID).To(BeEmpty())
|
||||
Expect(mf.SortArtistName).To(BeEmpty())
|
||||
|
||||
By("Makingsure the artist in the DB has not changed")
|
||||
aa, err = ds.Artist(ctx).GetAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"name": "The Beatles"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(aa).To(HaveLen(1))
|
||||
Expect(aa[0].Name).To(Equal("The Beatles"))
|
||||
Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID))
|
||||
Expect(aa[0].SortArtistName).To(Equal("Beatles, The"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func createFindByPath(ctx context.Context, ds model.DataStore) func(string) (*model.MediaFile, error) {
|
||||
return func(path string) (*model.MediaFile, error) {
|
||||
list, err := ds.MediaFile(ctx).FindByPaths([]string{path})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
return &list[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
type mockMediaFileRepo struct {
|
||||
model.MediaFileRepository
|
||||
GetMissingAndMatchingError error
|
||||
}
|
||||
|
||||
func (m *mockMediaFileRepo) GetMissingAndMatching(libId int) (model.MediaFileCursor, error) {
|
||||
if m.GetMissingAndMatchingError != nil {
|
||||
return nil, m.GetMissingAndMatchingError
|
||||
}
|
||||
return m.MediaFileRepository.GetMissingAndMatching(libId)
|
||||
}
|
||||
|
|
@ -1,440 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/auth"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/scanner/metadata"
|
||||
_ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg"
|
||||
_ "github.com/navidrome/navidrome/scanner/metadata/taglib"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type TagScanner struct {
|
||||
// Dependencies
|
||||
ds model.DataStore
|
||||
playlists core.Playlists
|
||||
cacheWarmer artwork.CacheWarmer
|
||||
|
||||
// Internal state
|
||||
lib model.Library
|
||||
cnt *counters
|
||||
mapper *MediaFileMapper
|
||||
}
|
||||
|
||||
func NewTagScanner(ds model.DataStore, playlists core.Playlists, cacheWarmer artwork.CacheWarmer) FolderScanner {
|
||||
s := &TagScanner{
|
||||
ds: ds,
|
||||
cacheWarmer: cacheWarmer,
|
||||
playlists: playlists,
|
||||
}
|
||||
metadata.LogExtractors()
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
type dirMap map[string]dirStats
|
||||
|
||||
type counters struct {
|
||||
added int64
|
||||
updated int64
|
||||
deleted int64
|
||||
playlists int64
|
||||
}
|
||||
|
||||
func (cnt *counters) total() int64 { return cnt.added + cnt.updated + cnt.deleted }
|
||||
|
||||
const (
|
||||
// filesBatchSize used for batching file metadata extraction
|
||||
filesBatchSize = 100
|
||||
)
|
||||
|
||||
// Scan algorithm overview:
|
||||
// Load all directories from the DB
|
||||
// Traverse the music folder, collecting each subfolder's ModTime (self or any non-dir children, whichever is newer)
|
||||
// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
|
||||
// - if file in folder is newer, update the one in DB
|
||||
// - if file in folder does not exists in DB, add it
|
||||
// - for each file in the DB that is not found in the folder, delete it from DB
|
||||
// Compare directories in the fs with the ones in the DB to find deleted folders
|
||||
// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
|
||||
// Create new albums/artists, update counters:
|
||||
// - collect all albumIDs and artistIDs from previous steps
|
||||
// - refresh the collected albums and artists with the metadata from the mediafiles
|
||||
// For each changed folder, process playlists:
|
||||
// - If the playlist is not in the DB, import it, setting sync = true
|
||||
// - If the playlist is in the DB and sync == true, import it, or else skip it
|
||||
// Delete all empty albums, delete all empty artists, clean-up playlists
|
||||
func (s *TagScanner) Scan(ctx context.Context, lib model.Library, fullScan bool, progress chan uint32) (int64, error) {
|
||||
ctx = auth.WithAdminUser(ctx, s.ds)
|
||||
start := time.Now()
|
||||
|
||||
// Update internal copy of Library
|
||||
s.lib = lib
|
||||
|
||||
// Special case: if LastScanAt is zero, re-import all files
|
||||
fullScan = fullScan || s.lib.LastScanAt.IsZero()
|
||||
|
||||
// If the media folder is empty (no music and no subfolders), abort to avoid deleting all data from DB
|
||||
empty, err := isDirEmpty(ctx, s.lib.Path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if empty && !fullScan {
|
||||
log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.lib.Path)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
allDBDirs, err := s.getDBDirTree(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
allFSDirs := dirMap{}
|
||||
var changedDirs []string
|
||||
s.cnt = &counters{}
|
||||
genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx))
|
||||
s.mapper = NewMediaFileMapper(s.lib.Path, genres)
|
||||
refresher := newRefresher(s.ds, s.cacheWarmer, s.lib, allFSDirs)
|
||||
|
||||
log.Trace(ctx, "Loading directory tree from music folder", "folder", s.lib.Path)
|
||||
foldersFound, walkerError := walkDirTree(ctx, s.lib.Path)
|
||||
|
||||
// Process each folder found in the music folder
|
||||
g, walkCtx := errgroup.WithContext(ctx)
|
||||
g.Go(func() error {
|
||||
for folderStats := range pl.ReadOrDone(walkCtx, foldersFound) {
|
||||
updateProgress(progress, folderStats.AudioFilesCount)
|
||||
allFSDirs[folderStats.Path] = folderStats
|
||||
|
||||
if s.folderHasChanged(folderStats, allDBDirs, s.lib.LastScanAt) || fullScan {
|
||||
changedDirs = append(changedDirs, folderStats.Path)
|
||||
log.Debug("Processing changed folder", "dir", folderStats.Path)
|
||||
err := s.processChangedDir(walkCtx, refresher, fullScan, folderStats.Path)
|
||||
if err != nil {
|
||||
log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// Check for errors in the walker
|
||||
g.Go(func() error {
|
||||
for err := range walkerError {
|
||||
log.Error("Scan was interrupted by error. See errors above", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// Wait for all goroutines to finish, and check if an error occurred
|
||||
if err := g.Wait(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
|
||||
if len(deletedDirs)+len(changedDirs) == 0 {
|
||||
log.Debug(ctx, "No changes found in Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start))
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
for _, dir := range deletedDirs {
|
||||
err := s.processDeletedDir(ctx, refresher, dir)
|
||||
if err != nil {
|
||||
log.Error("Error removing deleted folder from DB", "dir", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
s.cnt.playlists = 0
|
||||
if conf.Server.AutoImportPlaylists {
|
||||
// Now that all mediafiles are imported/updated, search for and import/update playlists
|
||||
u, _ := request.UserFrom(ctx)
|
||||
for _, dir := range changedDirs {
|
||||
info := allFSDirs[dir]
|
||||
if info.HasPlaylist {
|
||||
if !u.IsAdmin {
|
||||
log.Warn("Playlists will not be imported, as there are no admin users yet, "+
|
||||
"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
|
||||
} else {
|
||||
plsSync := newPlaylistImporter(s.ds, s.playlists, s.cacheWarmer, lib.Path)
|
||||
s.cnt.playlists = plsSync.processPlaylists(ctx, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Debug("Playlist auto-import is disabled")
|
||||
}
|
||||
|
||||
err = s.ds.GC(log.NewContext(ctx), s.lib.Path)
|
||||
log.Info("Finished processing Music Folder", "folder", s.lib.Path, "elapsed", time.Since(start),
|
||||
"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists)
|
||||
|
||||
return s.cnt.total(), err
|
||||
}
|
||||
|
||||
func updateProgress(progress chan uint32, count uint32) {
|
||||
select {
|
||||
case progress <- count:
|
||||
default: // It is ok to miss a count update
|
||||
}
|
||||
}
|
||||
|
||||
func isDirEmpty(ctx context.Context, dir string) (bool, error) {
|
||||
children, stats, err := loadDir(ctx, dir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return len(children) == 0 && stats.AudioFilesCount == 0, nil
|
||||
}
|
||||
|
||||
func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
|
||||
start := time.Now()
|
||||
log.Trace(ctx, "Loading directory tree from database", "folder", s.lib.Path)
|
||||
|
||||
repo := s.ds.MediaFile(ctx)
|
||||
dirs, err := repo.FindPathsRecursively(s.lib.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp := map[string]struct{}{}
|
||||
for _, d := range dirs {
|
||||
resp[filepath.Clean(d)] = struct{}{}
|
||||
}
|
||||
|
||||
log.Debug("Directory tree loaded from DB", "total", len(resp), "elapsed", time.Since(start))
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *TagScanner) folderHasChanged(folder dirStats, dbDirs map[string]struct{}, lastModified time.Time) bool {
|
||||
_, inDB := dbDirs[folder.Path]
|
||||
// If is a new folder with at least one song OR it was modified after lastModified
|
||||
return (!inDB && (folder.AudioFilesCount > 0)) || folder.ModTime.After(lastModified)
|
||||
}
|
||||
|
||||
func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {
|
||||
start := time.Now()
|
||||
log.Trace(ctx, "Checking for deleted folders")
|
||||
var deleted []string
|
||||
|
||||
for d := range dbDirs {
|
||||
if _, ok := fsDirs[d]; !ok {
|
||||
deleted = append(deleted, d)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(deleted)
|
||||
log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
|
||||
return deleted
|
||||
}
|
||||
|
||||
func (s *TagScanner) processDeletedDir(ctx context.Context, refresher *refresher, dir string) error {
|
||||
start := time.Now()
|
||||
|
||||
mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.cnt.deleted += c
|
||||
|
||||
for _, t := range mfs {
|
||||
refresher.accumulate(t)
|
||||
}
|
||||
|
||||
err = refresher.flush(ctx)
|
||||
log.Info(ctx, "Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *TagScanner) processChangedDir(ctx context.Context, refresher *refresher, fullScan bool, dir string) error {
|
||||
start := time.Now()
|
||||
|
||||
// Load folder's current tracks from DB into a map
|
||||
currentTracks := map[string]model.MediaFile{}
|
||||
ct, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range ct {
|
||||
currentTracks[t.Path] = t
|
||||
}
|
||||
|
||||
// Load track list from the folder
|
||||
files, err := loadAllAudioFiles(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If no files to process, return
|
||||
if len(files)+len(currentTracks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
orphanTracks := map[string]model.MediaFile{}
|
||||
for k, v := range currentTracks {
|
||||
orphanTracks[k] = v
|
||||
}
|
||||
|
||||
// If track from folder is newer than the one in DB, select for update/insert in DB
|
||||
log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
|
||||
filesToUpdate := make([]string, 0, len(files))
|
||||
for filePath, entry := range files {
|
||||
c, inDB := currentTracks[filePath]
|
||||
if !inDB || fullScan {
|
||||
filesToUpdate = append(filesToUpdate, filePath)
|
||||
s.cnt.added++
|
||||
} else {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
log.Error("Could not stat file", "filePath", filePath, err)
|
||||
continue
|
||||
}
|
||||
if info.ModTime().After(c.UpdatedAt) {
|
||||
filesToUpdate = append(filesToUpdate, filePath)
|
||||
s.cnt.updated++
|
||||
}
|
||||
}
|
||||
|
||||
// Force a refresh of the album and artist, to cater for cover art files
|
||||
refresher.accumulate(c)
|
||||
|
||||
// Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks
|
||||
// are considered gone from the music folder and will be deleted from DB
|
||||
delete(orphanTracks, filePath)
|
||||
}
|
||||
|
||||
numUpdatedTracks := 0
|
||||
numPurgedTracks := 0
|
||||
|
||||
if len(filesToUpdate) > 0 {
|
||||
numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, refresher, dir, currentTracks, filesToUpdate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(orphanTracks) > 0 {
|
||||
numPurgedTracks, err = s.deleteOrphanSongs(ctx, refresher, dir, orphanTracks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = refresher.flush(ctx)
|
||||
log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
|
||||
"deleted", numPurgedTracks, "elapsed", time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *TagScanner) deleteOrphanSongs(
|
||||
ctx context.Context,
|
||||
refresher *refresher,
|
||||
dir string,
|
||||
tracksToDelete map[string]model.MediaFile,
|
||||
) (int, error) {
|
||||
numPurgedTracks := 0
|
||||
|
||||
log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
|
||||
// Remaining tracks from DB that are not in the folder are deleted
|
||||
for _, ct := range tracksToDelete {
|
||||
numPurgedTracks++
|
||||
refresher.accumulate(ct)
|
||||
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.cnt.deleted++
|
||||
}
|
||||
return numPurgedTracks, nil
|
||||
}
|
||||
|
||||
func (s *TagScanner) addOrUpdateTracksInDB(
|
||||
ctx context.Context,
|
||||
refresher *refresher,
|
||||
dir string,
|
||||
currentTracks map[string]model.MediaFile,
|
||||
filesToUpdate []string,
|
||||
) (int, error) {
|
||||
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
|
||||
|
||||
numUpdatedTracks := 0
|
||||
// Break the file list in chunks to avoid calling ffmpeg with too many parameters
|
||||
for chunk := range slices.Chunk(filesToUpdate, filesBatchSize) {
|
||||
// Load tracks Metadata from the folder
|
||||
newTracks, err := s.loadTracks(chunk)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// If track from folder is newer than the one in DB, update/insert in DB
|
||||
log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
|
||||
for i := range newTracks {
|
||||
n := newTracks[i]
|
||||
// Keep current annotations if the track is in the DB
|
||||
if t, ok := currentTracks[n.Path]; ok {
|
||||
n.Annotations = t.Annotations
|
||||
}
|
||||
n.LibraryID = s.lib.ID
|
||||
err := s.ds.MediaFile(ctx).Put(&n)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
refresher.accumulate(n)
|
||||
numUpdatedTracks++
|
||||
}
|
||||
}
|
||||
return numUpdatedTracks, nil
|
||||
}
|
||||
|
||||
func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
|
||||
mds, err := metadata.Extract(filePaths...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var mfs model.MediaFiles
|
||||
for _, md := range mds {
|
||||
mf := s.mapper.ToMediaFile(md)
|
||||
mfs = append(mfs, mf)
|
||||
}
|
||||
return mfs, nil
|
||||
}
|
||||
|
||||
func loadAllAudioFiles(dirPath string) (map[string]fs.DirEntry, error) {
|
||||
files, err := fs.ReadDir(os.DirFS(dirPath), ".")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fileInfos := make(map[string]fs.DirEntry)
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(f.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
filePath := filepath.Join(dirPath, f.Name())
|
||||
if !model.IsAudioFile(filePath) {
|
||||
continue
|
||||
}
|
||||
fileInfos[filePath] = f
|
||||
}
|
||||
|
||||
return fileInfos, nil
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("TagScanner", func() {
|
||||
Describe("loadAllAudioFiles", func() {
|
||||
It("return all audio files from the folder", func() {
|
||||
files, err := loadAllAudioFiles("tests/fixtures")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(files).To(HaveLen(11))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.aiff"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.flac"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.m4a"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.mp3"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.tak"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.wav"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.wma"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/test.wv"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.mp3"))
|
||||
Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.m4a"))
|
||||
Expect(files).ToNot(HaveKey("tests/fixtures/._02 Invisible.mp3"))
|
||||
Expect(files).ToNot(HaveKey("tests/fixtures/playlist.m3u"))
|
||||
})
|
||||
|
||||
It("returns error if path does not exist", func() {
|
||||
_, err := loadAllAudioFiles("./INVALID/PATH")
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns empty map if there are no audio files in path", func() {
|
||||
Expect(loadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,129 +1,239 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/chrono"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
type (
|
||||
dirStats struct {
|
||||
Path string
|
||||
ModTime time.Time
|
||||
Images []string
|
||||
ImagesUpdatedAt time.Time
|
||||
HasPlaylist bool
|
||||
AudioFilesCount uint32
|
||||
}
|
||||
)
|
||||
|
||||
func walkDirTree(ctx context.Context, rootFolder string) (<-chan dirStats, <-chan error) {
|
||||
results := make(chan dirStats)
|
||||
errC := make(chan error)
|
||||
go func() {
|
||||
defer close(results)
|
||||
defer close(errC)
|
||||
err := walkFolder(ctx, rootFolder, rootFolder, results)
|
||||
if err != nil {
|
||||
log.Error(ctx, "There were errors reading directories from filesystem", "path", rootFolder, err)
|
||||
errC <- err
|
||||
}
|
||||
log.Debug(ctx, "Finished reading directories from filesystem", "path", rootFolder)
|
||||
}()
|
||||
return results, errC
|
||||
type folderEntry struct {
|
||||
job *scanJob
|
||||
elapsed chrono.Meter
|
||||
path string // Full path
|
||||
id string // DB ID
|
||||
modTime time.Time // From FS
|
||||
updTime time.Time // from DB
|
||||
audioFiles map[string]fs.DirEntry
|
||||
imageFiles map[string]fs.DirEntry
|
||||
numPlaylists int
|
||||
numSubFolders int
|
||||
imagesUpdatedAt time.Time
|
||||
tracks model.MediaFiles
|
||||
albums model.Albums
|
||||
albumIDMap map[string]string
|
||||
artists model.Artists
|
||||
tags model.TagList
|
||||
missingTracks []*model.MediaFile
|
||||
}
|
||||
|
||||
func walkFolder(ctx context.Context, rootPath string, currentFolder string, results chan<- dirStats) error {
|
||||
children, stats, err := loadDir(ctx, currentFolder)
|
||||
func (f *folderEntry) hasNoFiles() bool {
|
||||
return len(f.audioFiles) == 0 && len(f.imageFiles) == 0 && f.numPlaylists == 0 && f.numSubFolders == 0
|
||||
}
|
||||
|
||||
func (f *folderEntry) isNew() bool {
|
||||
return f.updTime.IsZero()
|
||||
}
|
||||
|
||||
func (f *folderEntry) toFolder() *model.Folder {
|
||||
folder := model.NewFolder(f.job.lib, f.path)
|
||||
folder.NumAudioFiles = len(f.audioFiles)
|
||||
if core.InPlaylistsPath(*folder) {
|
||||
folder.NumPlaylists = f.numPlaylists
|
||||
}
|
||||
folder.ImageFiles = slices.Collect(maps.Keys(f.imageFiles))
|
||||
folder.ImagesUpdatedAt = f.imagesUpdatedAt
|
||||
return folder
|
||||
}
|
||||
|
||||
func newFolderEntry(job *scanJob, path string) *folderEntry {
|
||||
id := model.FolderID(job.lib, path)
|
||||
f := &folderEntry{
|
||||
id: id,
|
||||
job: job,
|
||||
path: path,
|
||||
audioFiles: make(map[string]fs.DirEntry),
|
||||
imageFiles: make(map[string]fs.DirEntry),
|
||||
albumIDMap: make(map[string]string),
|
||||
updTime: job.popLastUpdate(id),
|
||||
}
|
||||
f.elapsed.Start()
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *folderEntry) isOutdated() bool {
|
||||
if f.job.lib.FullScanInProgress {
|
||||
return f.updTime.Before(f.job.lib.LastScanStartedAt)
|
||||
}
|
||||
return f.updTime.Before(f.modTime)
|
||||
}
|
||||
|
||||
func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) {
|
||||
results := make(chan *folderEntry)
|
||||
go func() {
|
||||
defer close(results)
|
||||
err := walkFolder(ctx, job, ".", nil, results)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err)
|
||||
return
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Finished reading folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load())
|
||||
}()
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error {
|
||||
ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns)
|
||||
|
||||
folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns)
|
||||
if err != nil {
|
||||
return err
|
||||
log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err)
|
||||
return nil
|
||||
}
|
||||
for _, c := range children {
|
||||
err := walkFolder(ctx, rootPath, c, results)
|
||||
err := walkFolder(ctx, job, c, ignorePatterns, results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
dir := filepath.Clean(currentFolder)
|
||||
log.Trace(ctx, "Found directory", "dir", dir, "audioCount", stats.AudioFilesCount,
|
||||
"images", stats.Images, "hasPlaylist", stats.HasPlaylist)
|
||||
stats.Path = dir
|
||||
results <- *stats
|
||||
dir := path.Clean(currentFolder)
|
||||
log.Trace(ctx, "Scanner: Found directory", " path", dir, "audioFiles", maps.Keys(folder.audioFiles),
|
||||
"images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt,
|
||||
"updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children))
|
||||
folder.path = dir
|
||||
results <- folder
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) {
|
||||
stats := &dirStats{}
|
||||
func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string {
|
||||
ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile)
|
||||
var newPatterns []string
|
||||
if _, err := fs.Stat(fsys, ignoreFilePath); err == nil {
|
||||
// Read and parse the .ndignore file
|
||||
ignoreFile, err := fsys.Open(ignoreFilePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
|
||||
// Continue with previous patterns
|
||||
} else {
|
||||
defer ignoreFile.Close()
|
||||
scanner := bufio.NewScanner(ignoreFile)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // Skip empty lines and comments
|
||||
}
|
||||
newPatterns = append(newPatterns, line)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Warn(ctx, "Scanner: Error reading .ignore file", "path", ignoreFilePath, err)
|
||||
}
|
||||
}
|
||||
// If the .ndignore file is empty, mimic the current behavior and ignore everything
|
||||
if len(newPatterns) == 0 {
|
||||
newPatterns = []string{"**/*"}
|
||||
}
|
||||
}
|
||||
// Combine the patterns from the .ndignore file with the ones passed as argument
|
||||
combinedPatterns := append([]string{}, currentPatterns...)
|
||||
return append(combinedPatterns, newPatterns...)
|
||||
}
|
||||
|
||||
dirInfo, err := os.Stat(dirPath)
|
||||
func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) {
|
||||
folder = newFolderEntry(job, dirPath)
|
||||
|
||||
dirInfo, err := fs.Stat(job.fs, dirPath)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error stating dir", "path", dirPath, err)
|
||||
log.Warn(ctx, "Scanner: Error stating dir", "path", dirPath, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
stats.ModTime = dirInfo.ModTime()
|
||||
folder.modTime = dirInfo.ModTime()
|
||||
|
||||
dir, err := os.Open(dirPath)
|
||||
dir, err := job.fs.Open(dirPath)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error in Opening directory", "path", dirPath, err)
|
||||
return nil, stats, err
|
||||
log.Warn(ctx, "Scanner: Error in Opening directory", "path", dirPath, err)
|
||||
return folder, children, err
|
||||
}
|
||||
defer dir.Close()
|
||||
dirFile, ok := dir.(fs.ReadDirFile)
|
||||
if !ok {
|
||||
log.Error(ctx, "Not a directory", "path", dirPath)
|
||||
return folder, children, err
|
||||
}
|
||||
|
||||
entries := fullReadDir(ctx, dir)
|
||||
children := make([]string, 0, len(entries))
|
||||
ignoreMatcher := ignore.CompileIgnoreLines(ignorePatterns...)
|
||||
entries := fullReadDir(ctx, dirFile)
|
||||
children = make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
isDir, err := isDirOrSymlinkToDir(dirPath, entry)
|
||||
// Skip invalid symlinks
|
||||
if err != nil {
|
||||
log.Error(ctx, "Invalid symlink", "dir", filepath.Join(dirPath, entry.Name()), err)
|
||||
entryPath := path.Join(dirPath, entry.Name())
|
||||
if len(ignorePatterns) > 0 && isScanIgnored(ignoreMatcher, entryPath) {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)
|
||||
continue
|
||||
}
|
||||
if isDir && !isDirIgnored(dirPath, entry) && isDirReadable(ctx, dirPath, entry) {
|
||||
children = append(children, filepath.Join(dirPath, entry.Name()))
|
||||
if isEntryIgnored(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return folder, children, ctx.Err()
|
||||
}
|
||||
isDir, err := isDirOrSymlinkToDir(job.fs, dirPath, entry)
|
||||
// Skip invalid symlinks
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Invalid symlink", "dir", entryPath, err)
|
||||
continue
|
||||
}
|
||||
if isDir && !isDirIgnored(entry.Name()) && isDirReadable(ctx, job.fs, entryPath) {
|
||||
children = append(children, entryPath)
|
||||
folder.numSubFolders++
|
||||
} else {
|
||||
fileInfo, err := entry.Info()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting fileInfo", "name", entry.Name(), err)
|
||||
return children, stats, err
|
||||
log.Warn(ctx, "Scanner: Error getting fileInfo", "name", entry.Name(), err)
|
||||
return folder, children, err
|
||||
}
|
||||
if fileInfo.ModTime().After(stats.ModTime) {
|
||||
stats.ModTime = fileInfo.ModTime()
|
||||
if fileInfo.ModTime().After(folder.modTime) {
|
||||
folder.modTime = fileInfo.ModTime()
|
||||
}
|
||||
switch {
|
||||
case model.IsAudioFile(entry.Name()):
|
||||
stats.AudioFilesCount++
|
||||
folder.audioFiles[entry.Name()] = entry
|
||||
case model.IsValidPlaylist(entry.Name()):
|
||||
stats.HasPlaylist = true
|
||||
folder.numPlaylists++
|
||||
case model.IsImageFile(entry.Name()):
|
||||
stats.Images = append(stats.Images, entry.Name())
|
||||
if fileInfo.ModTime().After(stats.ImagesUpdatedAt) {
|
||||
stats.ImagesUpdatedAt = fileInfo.ModTime()
|
||||
folder.imageFiles[entry.Name()] = entry
|
||||
if fileInfo.ModTime().After(folder.imagesUpdatedAt) {
|
||||
folder.imagesUpdatedAt = fileInfo.ModTime()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return children, stats, nil
|
||||
return folder, children, nil
|
||||
}
|
||||
|
||||
// fullReadDir reads all files in the folder, skipping the ones with errors.
|
||||
// It also detects when it is "stuck" with an error in the same directory over and over.
|
||||
// In this case, it stops and returns whatever it was able to read until it got stuck.
|
||||
// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850
|
||||
func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
|
||||
var allEntries []os.DirEntry
|
||||
func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []fs.DirEntry {
|
||||
var allEntries []fs.DirEntry
|
||||
var prevErrStr = ""
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
entries, err := dir.ReadDir(-1)
|
||||
allEntries = append(allEntries, entries...)
|
||||
if err == nil {
|
||||
|
|
@ -131,7 +241,7 @@ func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
|
|||
}
|
||||
log.Warn(ctx, "Skipping DirEntry", err)
|
||||
if prevErrStr == err.Error() {
|
||||
log.Error(ctx, "Duplicate DirEntry failure, bailing", err)
|
||||
log.Error(ctx, "Scanner: Duplicate DirEntry failure, bailing", err)
|
||||
break
|
||||
}
|
||||
prevErrStr = err.Error()
|
||||
|
|
@ -146,55 +256,60 @@ func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
|
|||
// sending a request to the operating system to follow the symbolic link.
|
||||
// originally copied from github.com/karrick/godirwalk, modified to use dirEntry for
|
||||
// efficiency for go 1.16 and beyond
|
||||
func isDirOrSymlinkToDir(baseDir string, dirEnt fs.DirEntry) (bool, error) {
|
||||
func isDirOrSymlinkToDir(fsys fs.FS, baseDir string, dirEnt fs.DirEntry) (bool, error) {
|
||||
if dirEnt.IsDir() {
|
||||
return true, nil
|
||||
}
|
||||
if dirEnt.Type()&os.ModeSymlink == 0 {
|
||||
if dirEnt.Type()&fs.ModeSymlink == 0 {
|
||||
return false, nil
|
||||
}
|
||||
// Does this symlink point to a directory?
|
||||
fileInfo, err := os.Stat(filepath.Join(baseDir, dirEnt.Name()))
|
||||
fileInfo, err := fs.Stat(fsys, path.Join(baseDir, dirEnt.Name()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return fileInfo.IsDir(), nil
|
||||
}
|
||||
|
||||
// isDirReadable returns true if the directory represented by dirEnt is readable
|
||||
func isDirReadable(ctx context.Context, fsys fs.FS, dirPath string) bool {
|
||||
dir, err := fsys.Open(dirPath)
|
||||
if err != nil {
|
||||
log.Warn("Scanner: Skipping unreadable directory", "path", dirPath, err)
|
||||
return false
|
||||
}
|
||||
err = dir.Close()
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error closing directory", "path", dirPath, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// List of special directories to ignore
|
||||
var ignoredDirs = []string{
|
||||
"$RECYCLE.BIN",
|
||||
"#snapshot",
|
||||
"@Recently-Snapshot",
|
||||
".streams",
|
||||
"lost+found",
|
||||
}
|
||||
|
||||
// isDirIgnored returns true if the directory represented by dirEnt contains an
|
||||
// `ignore` file (named after skipScanFile)
|
||||
func isDirIgnored(baseDir string, dirEnt fs.DirEntry) bool {
|
||||
// isDirIgnored returns true if the directory represented by dirEnt should be ignored
|
||||
func isDirIgnored(name string) bool {
|
||||
// allows Album folders for albums which eg start with ellipses
|
||||
name := dirEnt.Name()
|
||||
if strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") {
|
||||
return true
|
||||
}
|
||||
if slices.IndexFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) }) != -1 {
|
||||
if slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) }) {
|
||||
return true
|
||||
}
|
||||
_, err := os.Stat(filepath.Join(baseDir, name, consts.SkipScanFile))
|
||||
return err == nil
|
||||
return false
|
||||
}
|
||||
|
||||
// isDirReadable returns true if the directory represented by dirEnt is readable
|
||||
func isDirReadable(ctx context.Context, baseDir string, dirEnt os.DirEntry) bool {
|
||||
path := filepath.Join(baseDir, dirEnt.Name())
|
||||
|
||||
dir, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Warn("Skipping unreadable directory", "path", path, err)
|
||||
return false
|
||||
}
|
||||
|
||||
err = dir.Close()
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Error closing directory", "path", path, err)
|
||||
}
|
||||
|
||||
return true
|
||||
func isEntryIgnored(name string) bool {
|
||||
return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
|
||||
}
|
||||
|
||||
func isScanIgnored(matcher *ignore.GitIgnore, entryPath string) bool {
|
||||
return matcher.MatchesPath(entryPath)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,87 +8,112 @@ import (
|
|||
"path/filepath"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
. "github.com/onsi/gomega/gstruct"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var _ = Describe("walk_dir_tree", func() {
|
||||
dir, _ := os.Getwd()
|
||||
baseDir := filepath.Join(dir, "tests", "fixtures")
|
||||
|
||||
Describe("walkDirTree", func() {
|
||||
It("reads all info correctly", func() {
|
||||
var collected = dirMap{}
|
||||
results, errC := walkDirTree(context.Background(), baseDir)
|
||||
|
||||
for {
|
||||
stats, more := <-results
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
collected[stats.Path] = stats
|
||||
var fsys storage.MusicFS
|
||||
BeforeEach(func() {
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"root/a/.ndignore": {Data: []byte("ignored/*")},
|
||||
"root/a/f1.mp3": {},
|
||||
"root/a/f2.mp3": {},
|
||||
"root/a/ignored/bad.mp3": {},
|
||||
"root/b/cover.jpg": {},
|
||||
"root/c/f3": {},
|
||||
"root/d": {},
|
||||
"root/d/.ndignore": {},
|
||||
"root/d/f1.mp3": {},
|
||||
"root/d/f2.mp3": {},
|
||||
"root/d/f3.mp3": {},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
Consistently(errC).ShouldNot(Receive())
|
||||
Expect(collected[baseDir]).To(MatchFields(IgnoreExtras, Fields{
|
||||
"Images": BeEmpty(),
|
||||
"HasPlaylist": BeFalse(),
|
||||
"AudioFilesCount": BeNumerically("==", 12),
|
||||
}))
|
||||
Expect(collected[filepath.Join(baseDir, "artist", "an-album")]).To(MatchFields(IgnoreExtras, Fields{
|
||||
"Images": ConsistOf("cover.jpg", "front.png", "artist.png"),
|
||||
"HasPlaylist": BeFalse(),
|
||||
"AudioFilesCount": BeNumerically("==", 1),
|
||||
}))
|
||||
Expect(collected[filepath.Join(baseDir, "playlists")].HasPlaylist).To(BeTrue())
|
||||
Expect(collected).To(HaveKey(filepath.Join(baseDir, "symlink2dir")))
|
||||
Expect(collected).To(HaveKey(filepath.Join(baseDir, "empty_folder")))
|
||||
It("walks all directories", func() {
|
||||
job := &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
ctx := context.Background()
|
||||
results, err := walkDirTree(ctx, job)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
|
||||
Expect(folders).To(HaveLen(6))
|
||||
Expect(folders["root/a/ignored"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
|
||||
HaveLen(2),
|
||||
HaveKey("f1.mp3"),
|
||||
HaveKey("f2.mp3"),
|
||||
))
|
||||
Expect(folders["root/a"].imageFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
|
||||
HaveLen(1),
|
||||
HaveKey("cover.jpg"),
|
||||
))
|
||||
Expect(folders["root/c"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/c"].imageFiles).To(BeEmpty())
|
||||
Expect(folders).ToNot(HaveKey("root/d"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("isDirOrSymlinkToDir", func() {
|
||||
It("returns true for normal dirs", func() {
|
||||
dirEntry := getDirEntry("tests", "fixtures")
|
||||
Expect(isDirOrSymlinkToDir(baseDir, dirEntry)).To(BeTrue())
|
||||
Describe("helper functions", func() {
|
||||
dir, _ := os.Getwd()
|
||||
fsys := os.DirFS(dir)
|
||||
baseDir := filepath.Join("tests", "fixtures")
|
||||
|
||||
Describe("isDirOrSymlinkToDir", func() {
|
||||
It("returns true for normal dirs", func() {
|
||||
dirEntry := getDirEntry("tests", "fixtures")
|
||||
Expect(isDirOrSymlinkToDir(fsys, baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns true for symlinks to dirs", func() {
|
||||
dirEntry := getDirEntry(baseDir, "symlink2dir")
|
||||
Expect(isDirOrSymlinkToDir(fsys, baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns false for files", func() {
|
||||
dirEntry := getDirEntry(baseDir, "test.mp3")
|
||||
Expect(isDirOrSymlinkToDir(fsys, baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
It("returns false for symlinks to files", func() {
|
||||
dirEntry := getDirEntry(baseDir, "symlink")
|
||||
Expect(isDirOrSymlinkToDir(fsys, baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
})
|
||||
It("returns true for symlinks to dirs", func() {
|
||||
dirEntry := getDirEntry(baseDir, "symlink2dir")
|
||||
Expect(isDirOrSymlinkToDir(baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns false for files", func() {
|
||||
dirEntry := getDirEntry(baseDir, "test.mp3")
|
||||
Expect(isDirOrSymlinkToDir(baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
It("returns false for symlinks to files", func() {
|
||||
dirEntry := getDirEntry(baseDir, "symlink")
|
||||
Expect(isDirOrSymlinkToDir(baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
})
|
||||
Describe("isDirIgnored", func() {
|
||||
It("returns false for normal dirs", func() {
|
||||
dirEntry := getDirEntry(baseDir, "empty_folder")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
It("returns true when folder contains .ndignore file", func() {
|
||||
dirEntry := getDirEntry(baseDir, "ignored_folder")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns true when folder name starts with a `.`", func() {
|
||||
dirEntry := getDirEntry(baseDir, ".hidden_folder")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns false when folder name starts with ellipses", func() {
|
||||
dirEntry := getDirEntry(baseDir, "...unhidden_folder")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeFalse())
|
||||
})
|
||||
It("returns true when folder name is $Recycle.Bin", func() {
|
||||
dirEntry := getDirEntry(baseDir, "$Recycle.Bin")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeTrue())
|
||||
})
|
||||
It("returns true when folder name is #snapshot", func() {
|
||||
dirEntry := getDirEntry(baseDir, "#snapshot")
|
||||
Expect(isDirIgnored(baseDir, dirEntry)).To(BeTrue())
|
||||
Describe("isDirIgnored", func() {
|
||||
It("returns false for normal dirs", func() {
|
||||
Expect(isDirIgnored("empty_folder")).To(BeFalse())
|
||||
})
|
||||
It("returns true when folder name starts with a `.`", func() {
|
||||
Expect(isDirIgnored(".hidden_folder")).To(BeTrue())
|
||||
})
|
||||
It("returns false when folder name starts with ellipses", func() {
|
||||
Expect(isDirIgnored("...unhidden_folder")).To(BeFalse())
|
||||
})
|
||||
It("returns true when folder name is $Recycle.Bin", func() {
|
||||
Expect(isDirIgnored("$Recycle.Bin")).To(BeTrue())
|
||||
})
|
||||
It("returns true when folder name is #snapshot", func() {
|
||||
Expect(isDirIgnored("#snapshot")).To(BeTrue())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -148,7 +173,7 @@ type fakeDirFile struct {
|
|||
}
|
||||
|
||||
// Only works with n == -1
|
||||
func (fd *fakeDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
|
||||
func (fd *fakeDirFile) ReadDir(int) ([]fs.DirEntry, error) {
|
||||
if fd.err != nil {
|
||||
return nil, fd.err
|
||||
}
|
||||
|
|
@ -179,3 +204,12 @@ func getDirEntry(baseDir, name string) os.DirEntry {
|
|||
}
|
||||
panic(fmt.Sprintf("Could not find %s in %s", name, baseDir))
|
||||
}
|
||||
|
||||
type mockMusicFS struct {
|
||||
storage.MusicFS
|
||||
fs.FS
|
||||
}
|
||||
|
||||
func (m *mockMusicFS) Open(name string) (fs.File, error) {
|
||||
return m.FS.Open(name)
|
||||
}
|
||||
|
|
|
|||
140
scanner/watcher.go
Normal file
140
scanner/watcher.go
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/storage"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type Watcher interface {
|
||||
Run(ctx context.Context) error
|
||||
}
|
||||
|
||||
type watcher struct {
|
||||
ds model.DataStore
|
||||
scanner Scanner
|
||||
triggerWait time.Duration
|
||||
}
|
||||
|
||||
func NewWatcher(ds model.DataStore, s Scanner) Watcher {
|
||||
return &watcher{ds: ds, scanner: s, triggerWait: conf.Server.Scanner.WatcherWait}
|
||||
}
|
||||
|
||||
func (w *watcher) Run(ctx context.Context) error {
|
||||
libs, err := w.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting libraries: %w", err)
|
||||
}
|
||||
|
||||
watcherChan := make(chan struct{})
|
||||
defer close(watcherChan)
|
||||
|
||||
// Start a watcher for each library
|
||||
for _, lib := range libs {
|
||||
go watchLib(ctx, lib, watcherChan)
|
||||
}
|
||||
|
||||
trigger := time.NewTimer(w.triggerWait)
|
||||
trigger.Stop()
|
||||
waiting := false
|
||||
for {
|
||||
select {
|
||||
case <-trigger.C:
|
||||
log.Info("Watcher: Triggering scan")
|
||||
status, err := w.scanner.Status(ctx)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error retrieving Scanner status", err)
|
||||
break
|
||||
}
|
||||
if status.Scanning {
|
||||
log.Debug(ctx, "Watcher: Already scanning, will retry later", "waitTime", w.triggerWait*3)
|
||||
trigger.Reset(w.triggerWait * 3)
|
||||
continue
|
||||
}
|
||||
waiting = false
|
||||
go func() {
|
||||
_, err := w.scanner.ScanAll(ctx, false)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error scanning", err)
|
||||
} else {
|
||||
log.Info(ctx, "Watcher: Scan completed")
|
||||
}
|
||||
}()
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-watcherChan:
|
||||
if !waiting {
|
||||
log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan")
|
||||
waiting = true
|
||||
}
|
||||
|
||||
trigger.Reset(w.triggerWait)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func watchLib(ctx context.Context, lib model.Library, watchChan chan struct{}) {
|
||||
s, err := storage.For(lib.Path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error creating storage", "library", lib.ID, "path", lib.Path, err)
|
||||
return
|
||||
}
|
||||
fsys, err := s.FS()
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error getting FS", "library", lib.ID, "path", lib.Path, err)
|
||||
return
|
||||
}
|
||||
watcher, ok := s.(storage.Watcher)
|
||||
if !ok {
|
||||
log.Info(ctx, "Watcher not supported", "library", lib.ID, "path", lib.Path)
|
||||
return
|
||||
}
|
||||
c, err := watcher.Start(ctx)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error watching library", "library", lib.ID, "path", lib.Path, err)
|
||||
return
|
||||
}
|
||||
log.Info(ctx, "Watcher started", "library", lib.ID, "path", lib.Path)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case path := <-c:
|
||||
path, err = filepath.Rel(lib.Path, path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error getting relative path", "library", lib.ID, "path", path, err)
|
||||
continue
|
||||
}
|
||||
if isIgnoredPath(ctx, fsys, path) {
|
||||
log.Trace(ctx, "Watcher: Ignoring change", "library", lib.ID, "path", path)
|
||||
continue
|
||||
}
|
||||
log.Trace(ctx, "Watcher: Detected change", "library", lib.ID, "path", path)
|
||||
watchChan <- struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
|
||||
baseDir, name := filepath.Split(path)
|
||||
switch {
|
||||
case model.IsAudioFile(path):
|
||||
return false
|
||||
case model.IsValidPlaylist(path):
|
||||
return false
|
||||
case model.IsImageFile(path):
|
||||
return false
|
||||
case name == ".DS_Store":
|
||||
return true
|
||||
}
|
||||
// As it can be a deletion and not a change, we cannot reliably know if the path is a file or directory.
|
||||
// But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway
|
||||
return isDirIgnored(baseDir)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue