feat(bfr): Big Refactor: new scanner, lots of new fields and tags, improvements and DB schema changes (#2709)

* fix(server): more race conditions when updating artist/album from external sources

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scanner): add .gitignore syntax to .ndignore. Resolves #1394

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): null

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): pass configfile option to child process

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): resume interrupted fullScans

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): remove old scanner code

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): rename old metadata package

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): move old metadata package

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: tests

Signed-off-by: Deluan <deluan@navidrome.org>

* chore(deps): update Go to 1.23.4

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: logs

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(test):

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: log level

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: remove log message

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: add config for scanner watcher

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: children playlists

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: replace `interface{}` with `any`

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: smart playlists with genres

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: allow any tags in smart playlists

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: artist names in playlists

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: smart playlist's sort by tags

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add moods to child

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add moods to AlbumID3

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(subsonic): use generic JSONArray for OS arrays

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(subsonic): use https in test

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add releaseTypes to AlbumID3

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add recordLabels to AlbumID3

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(subsonic): rename JSONArray to Array

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add artists to AlbumID3

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add artists to Child

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): do not pre-populate smart playlists

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): implement a simplified version of ArtistID3.

See https://github.com/opensubsonic/open-subsonic-api/discussions/120

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add artists to album child

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add contributors to mediafile Child

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add albumArtists to mediafile Child

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add displayArtist and displayAlbumArtist

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add displayComposer to Child

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add roles to ArtistID3

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(subsonic): use " • " separator for displayComposer

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor:

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(subsonic):

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(subsonic): respect `PreferSortTags` config option

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(subsonic):

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: optimize purging non-unused tags

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: don't run 'refresh artist stats' concurrently with other transactions

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor:

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: log message

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: add Scanner.ScanOnStartup config option, default true

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: better json parsing error msg when importing NSPs

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't update album's imported_time when updating external_metadata

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: handle interrupted scans and full scans after migrations

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: run `analyze` when migration requires a full rescan

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: run `PRAGMA optimize` at the end of the scan

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't update artist's updated_at when updating external_metadata

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: handle multiple artists and roles in smart playlists

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): dim missing tracks

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: album missing logic

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: error encoding in gob

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: separate warnings from errors

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: mark albums as missing if they were contained in a deleted folder

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: add participant names to media_file and album tables

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: use participations in criteria, instead of m2m relationship

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: rename participations to participants

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add moods to album child

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: albumartist role case

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scanner): run scanner as an external process by default

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): show albumArtist names

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): dim out missing albums

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: flaky test

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(server): scrobble buffer mapping. fix #3583

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: more participations renaming

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: listenbrainz scrobbling

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: send release_group_mbid to listenbrainz

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): implement OpenSubsonic explicitStatus field (#3597)

* feat: implement OpenSubsonic explicitStatus field

* fix(subsonic): fix failing snapshot tests

* refactor: create helper for setting explicitStatus

* fix: store smaller values for explicit-status on database

* test: ToAlbum explicitStatus

* refactor: rename explicitStatus helper function

---------

Co-authored-by: Deluan Quintão <deluan@navidrome.org>

* fix: handle album and track tags in the DB based on the mappings.yaml file

Signed-off-by: Deluan <deluan@navidrome.org>

* save similar artists as JSONB

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: getAlbumList byGenre

Signed-off-by: Deluan <deluan@navidrome.org>

* detect changes in PID configuration

Signed-off-by: Deluan <deluan@navidrome.org>

* set default album PID to legacy_pid

Signed-off-by: Deluan <deluan@navidrome.org>

* fix tests

Signed-off-by: Deluan <deluan@navidrome.org>

* fix SIGSEGV

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't lose album stars/ratings when migrating

Signed-off-by: Deluan <deluan@navidrome.org>

* store full PID conf in properties

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: keep album annotations when changing PID.Album config

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: reassign album annotations

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: use (display) albumArtist and add links to each artist

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: not showing albums by albumartist

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: error msgs

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: hide PID from Native API

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: album cover art resolution

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: trim participant names

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: reduce watcher log spam

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: panic when initializing the watcher

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: various artists

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't store empty lyrics in the DB

Signed-off-by: Deluan <deluan@navidrome.org>

* remove unused methods

Signed-off-by: Deluan <deluan@navidrome.org>

* drop full_text indexes, as they are not being used by SQLite

Signed-off-by: Deluan <deluan@navidrome.org>

* keep album created_at when upgrading

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): null pointer

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: album artwork cache

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't expose missing files in Subsonic API

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: searchable interface

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: filter out missing items from subsonic search

* fix: filter out missing items from playlists

* fix: filter out missing items from shares

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add filter by artist role

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): only return albumartists in getIndexes and getArtists endpoints

Signed-off-by: Deluan <deluan@navidrome.org>

* sort roles alphabetically

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: artist playcounts

Signed-off-by: Deluan <deluan@navidrome.org>

* change default Album PID conf

Signed-off-by: Deluan <deluan@navidrome.org>

* fix albumartist link when it does not match any albumartists values

Signed-off-by: Deluan <deluan@navidrome.org>

* fix `Ignoring filter not whitelisted` (role) message

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: trim any names/titles being imported

Signed-off-by: Deluan <deluan@navidrome.org>

* remove unused genre code

Signed-off-by: Deluan <deluan@navidrome.org>

* serialize calls to Last.fm's getArtist

Signed-off-by: Deluan <deluan@navidrome.org>

xxx

Signed-off-by: Deluan <deluan@navidrome.org>

* add counters to genres

Signed-off-by: Deluan <deluan@navidrome.org>

* nit: fix migration `notice` message

Signed-off-by: Deluan <deluan@navidrome.org>

* optimize similar artists query

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: last.fm.getInfo when mbid does not exist

Signed-off-by: Deluan <deluan@navidrome.org>

* ui only show missing items for admins

Signed-off-by: Deluan <deluan@navidrome.org>

* don't allow interaction with missing items

Signed-off-by: Deluan <deluan@navidrome.org>

* Add Missing Files view (WIP)

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: merged tag_counts into tag table

Signed-off-by: Deluan <deluan@navidrome.org>

* add option to completely disable automatic scanner

Signed-off-by: Deluan <deluan@navidrome.org>

* add delete missing files functionality

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: playlists not showing for regular users

Signed-off-by: Deluan <deluan@navidrome.org>

* reduce updateLastAccess frequency to once every minute

Signed-off-by: Deluan <deluan@navidrome.org>

* reduce update player frequency to once every minute

Signed-off-by: Deluan <deluan@navidrome.org>

* add timeout when updating player

Signed-off-by: Deluan <deluan@navidrome.org>

* remove dead code

Signed-off-by: Deluan <deluan@navidrome.org>

* fix duplicated roles in stats

Signed-off-by: Deluan <deluan@navidrome.org>

* add `; ` to artist splitters

Signed-off-by: Deluan <deluan@navidrome.org>

* fix stats query

Signed-off-by: Deluan <deluan@navidrome.org>

* more logs

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: support legacy clients (DSub) by removing OpenSubsonic extra fields - WIP

Signed-off-by: Deluan <deluan@navidrome.org>

* add record label filter

Signed-off-by: Deluan <deluan@navidrome.org>

* add release type filter

Signed-off-by: Deluan <deluan@navidrome.org>

* fix purgeUnused tags

Signed-off-by: Deluan <deluan@navidrome.org>

* add grouping filter to albums

Signed-off-by: Deluan <deluan@navidrome.org>

* allow any album tags to be used in as filters in the API

Signed-off-by: Deluan <deluan@navidrome.org>

* remove empty tags from album info

Signed-off-by: Deluan <deluan@navidrome.org>

* comments in the migration

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: Cannot read properties of undefined

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: listenbrainz scrobbling (#3640)

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: remove duplicated tag values

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: don't ignore the taglib folder!

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: show track subtitle tag

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: show artists stats based on selected role

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: inspect

Signed-off-by: Deluan <deluan@navidrome.org>

* add media type to album info/filters

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: change format of subtitle in the UI

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: subtitle in Subsonic API and search

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: subtitle in UI's player

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: split strings should be case-insensitive

Signed-off-by: Deluan <deluan@navidrome.org>

* disable ScanSchedule

Signed-off-by: Deluan <deluan@navidrome.org>

* increase default sessiontimeout

Signed-off-by: Deluan <deluan@navidrome.org>

* add sqlite command line tool to docker image

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: resources override

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: album PID conf

Signed-off-by: Deluan <deluan@navidrome.org>

* change migration to mark current artists as albumArtists

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): Allow filtering on multiple genres (#3679)

* feat(ui): Allow filtering on multiple genres

Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>

* add multi-genre filter in Album list

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>

* add more multi-valued tag filters to Album and Song views

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): unselect missing files after removing

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): song filter

Signed-off-by: Deluan <deluan@navidrome.org>

* fix sharing tracks. fix #3687

Signed-off-by: Deluan <deluan@navidrome.org>

* use rowids when using search for sync (ex: Symfonium)

Signed-off-by: Deluan <deluan@navidrome.org>

* fix "Report Real Paths" option for subsonic clients

Signed-off-by: Deluan <deluan@navidrome.org>

* fix "Report Real Paths" option for subsonic clients for search

Signed-off-by: Deluan <deluan@navidrome.org>

* add libraryPath to Native API /songs endpoint

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): add album version

Signed-off-by: Deluan <deluan@navidrome.org>

* made all tags lowercase as they are case-insensitive anyways.

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): Show full paths, extended properties for album/song (#3691)

* feat(ui): Show full paths, extended properties for album/song

- uses library path + os separator + path
- show participants (album/song) and tags (song)
- make album/participant clickable in show info

* add source to path

* fix pathSeparator in UI

Signed-off-by: Deluan <deluan@navidrome.org>

* fix local artist artwork (#3695)

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: parse vorbis performers

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: clean function into smaller functions

Signed-off-by: Deluan <deluan@navidrome.org>

* fix translations for en and pt

Signed-off-by: Deluan <deluan@navidrome.org>

* add trace log to show annotations reassignment

Signed-off-by: Deluan <deluan@navidrome.org>

* add trace log to show annotations reassignment

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: allow performers without instrument/subrole

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: metadata clean function again

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: optimize split function

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: split function is now a method of TagConf

Signed-off-by: Deluan <deluan@navidrome.org>

* fix: humanize Artist total size

Signed-off-by: Deluan <deluan@navidrome.org>

* add album version to album details

Signed-off-by: Deluan <deluan@navidrome.org>

* don't display album-level tags in SongInfo

Signed-off-by: Deluan <deluan@navidrome.org>

* fix genre clicking in Album Page

Signed-off-by: Deluan <deluan@navidrome.org>

* don't use mbids in Last.fm api calls.

From 1337574018:

With MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&mbid=a41ac10f-0a56-4672-9161-b83f9b223559&method=artist.getInfo

{
artist: {
name: "Bee Gees",
mbid: "bf0f7e29-dfe1-416c-b5c6-f9ebc19ea810",
url: "https://www.last.fm/music/Bee+Gees",
}
```

Without MBID:
```
GET https://ws.audioscrobbler.com/2.0/?api_key=XXXX&artist=Van+Morrison&format=json&lang=en&method=artist.getInfo

{
artist: {
name: "Van Morrison",
mbid: "a41ac10f-0a56-4672-9161-b83f9b223559",
url: "https://www.last.fm/music/Van+Morrison",
}
```

Signed-off-by: Deluan <deluan@navidrome.org>

* better logging for when the artist folder is not found

Signed-off-by: Deluan <deluan@navidrome.org>

* fix various issues with artist image resolution

Signed-off-by: Deluan <deluan@navidrome.org>

* hide "Additional Tags" header if there are none.

Signed-off-by: Deluan <deluan@navidrome.org>

* simplify tag rendering

Signed-off-by: Deluan <deluan@navidrome.org>

* enhance logging for artist folder detection

Signed-off-by: Deluan <deluan@navidrome.org>

* make folderID consistent for relative and absolute folderPaths

Signed-off-by: Deluan <deluan@navidrome.org>

* handle more folder paths scenarios

Signed-off-by: Deluan <deluan@navidrome.org>

* filter out other roles when SubsonicArtistParticipations = true

Signed-off-by: Deluan <deluan@navidrome.org>

* fix "Cannot read properties of undefined"

Signed-off-by: Deluan <deluan@navidrome.org>

* fix lyrics and comments being truncated (#3701)

* fix lyrics and comments being truncated

* specifically test for lyrics and comment length

* reorder assertions

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>

* fix(server): Expose library_path for playlist (#3705)

Allows showing absolute path for UI, and makes "report real path" work for playlists (Subsonic)

* fix BFR on Windows (#3704)

* fix potential reflected cross-site scripting vulnerability

Signed-off-by: Deluan <deluan@navidrome.org>

* hack to make it work on Windows

* ignore windows executables

* try fixing the pipeline

Signed-off-by: Deluan <deluan@navidrome.org>

* allow MusicFolder in other drives

* move windows local drive logic to local storage implementation

---------

Signed-off-by: Deluan <deluan@navidrome.org>

* increase pagination sizes for missing files

Signed-off-by: Deluan <deluan@navidrome.org>

* reduce level of "already scanning" watcher log message

Signed-off-by: Deluan <deluan@navidrome.org>

* only count folders with audio files in it

See https://github.com/navidrome/navidrome/discussions/3676#discussioncomment-11990930

Signed-off-by: Deluan <deluan@navidrome.org>

* add album version and catalog number to search

Signed-off-by: Deluan <deluan@navidrome.org>

* add `organization` alias for `recordlabel`

Signed-off-by: Deluan <deluan@navidrome.org>

* remove mbid from Last.fm agent

Signed-off-by: Deluan <deluan@navidrome.org>

* feat: support inspect in ui (#3726)

* inspect in ui

* address round 1

* add catalogNum to AlbumInfo

Signed-off-by: Deluan <deluan@navidrome.org>

* remove dependency on metadata_old (deprecated) package

Signed-off-by: Deluan <deluan@navidrome.org>

* add `RawTags` to model

Signed-off-by: Deluan <deluan@navidrome.org>

* support parsing MBIDs for roles (from the https://github.com/kgarner7/picard-all-mbids plugin) (#3698)


* parse standard roles, vorbis/m4a work for now

* fix djmixer

* working roles, use DJ-mix

* add performers to file

* map mbids

* add a few more tests

* add test

Signed-off-by: Deluan <deluan@navidrome.org>

* try to simplify the performers logic

Signed-off-by: Deluan <deluan@navidrome.org>

* stylistic changes

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Deluan <deluan@navidrome.org>

* remove param mutation

Signed-off-by: Deluan <deluan@navidrome.org>

* run automated SQLite optimizations

Signed-off-by: Deluan <deluan@navidrome.org>

* fix playlists import/export on Windows

* fix import playlists

* fix export playlists

* better handling of Windows volumes

Signed-off-by: Deluan <deluan@navidrome.org>

* handle more album ID reassignments

Signed-off-by: Deluan <deluan@navidrome.org>

* allow adding/overriding tags in the config file

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(ui): Fix playlist track id, handle missing tracks better (#3734)

- Use `mediaFileId` instead of `id` for playlist tracks
- Only fetch if the file is not missing
- If extractor fails to get the file, also error (rather than panic)

* optimize DB after each scan.

Signed-off-by: Deluan <deluan@navidrome.org>

* remove sortable from AlbumSongs columns

Signed-off-by: Deluan <deluan@navidrome.org>

* simplify query to get missing tracks

Signed-off-by: Deluan <deluan@navidrome.org>

* mark Scanner.Extractor as deprecated

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Signed-off-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Caio Cotts <caio@cotts.com.br>
Co-authored-by: Henrik Nordvik <henrikno@gmail.com>
Co-authored-by: Kendall Garner <17521368+kgarner7@users.noreply.github.com>
This commit is contained in:
Deluan Quintão 2025-02-19 17:35:17 -08:00 committed by GitHub
parent 46a963a02a
commit c795bcfcf7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
329 changed files with 16586 additions and 5852 deletions

View file

@ -0,0 +1,211 @@
package ffmpeg
import (
"bufio"
"context"
"errors"
"regexp"
"strconv"
"strings"
"time"
"github.com/navidrome/navidrome/core/ffmpeg"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/scanner/metadata_old"
)
const ExtractorID = "ffmpeg"
type Extractor struct {
ffmpeg ffmpeg.FFmpeg
}
func (e *Extractor) Parse(files ...string) (map[string]metadata_old.ParsedTags, error) {
output, err := e.ffmpeg.Probe(context.TODO(), files)
if err != nil {
log.Error("Cannot use ffmpeg to extract tags. Aborting", err)
return nil, err
}
fileTags := map[string]metadata_old.ParsedTags{}
if len(output) == 0 {
return fileTags, errors.New("error extracting metadata files")
}
infos := e.parseOutput(output)
for file, info := range infos {
tags, err := e.extractMetadata(file, info)
// Skip files with errors
if err == nil {
fileTags[file] = tags
}
}
return fileTags, nil
}
func (e *Extractor) CustomMappings() metadata_old.ParsedTags {
return metadata_old.ParsedTags{
"disc": {"tpa"},
"has_picture": {"metadata_block_picture"},
"originaldate": {"tdor"},
}
}
func (e *Extractor) Version() string {
return e.ffmpeg.Version()
}
func (e *Extractor) extractMetadata(filePath, info string) (metadata_old.ParsedTags, error) {
tags := e.parseInfo(info)
if len(tags) == 0 {
log.Trace("Not a media file. Skipping", "filePath", filePath)
return nil, errors.New("not a media file")
}
return tags, nil
}
var (
// Input #0, mp3, from 'groovin.mp3':
inputRegex = regexp.MustCompile(`(?m)^Input #\d+,.*,\sfrom\s'(.*)'`)
// TITLE : Back In Black
tagsRx = regexp.MustCompile(`(?i)^\s{4,6}([\w\s-]+)\s*:(.*)`)
// : Second comment line
continuationRx = regexp.MustCompile(`(?i)^\s+:(.*)`)
// Duration: 00:04:16.00, start: 0.000000, bitrate: 995 kb/s`
durationRx = regexp.MustCompile(`^\s\sDuration: ([\d.:]+).*bitrate: (\d+)`)
// Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s
bitRateRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+: Audio:.*, (\d+) kb/s`)
// Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s
// Stream #0:0: Audio: flac, 44100 Hz, stereo, s16
// Stream #0:0: Audio: dsd_lsbf_planar, 352800 Hz, stereo, fltp, 5644 kb/s
audioStreamRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+.*: Audio: (.*), (.*) Hz, ([\w.]+),*(.*.,)*`)
// Stream #0:1: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown), 600x600 [SAR 1:1 DAR 1:1], 90k tbr, 90k tbn, 90k tbc`
coverRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:.+: (Video):.*`)
)
func (e *Extractor) parseOutput(output string) map[string]string {
outputs := map[string]string{}
all := inputRegex.FindAllStringSubmatchIndex(output, -1)
for i, loc := range all {
// Filename is the first captured group
file := output[loc[2]:loc[3]]
// File info is everything from the match, up until the beginning of the next match
info := ""
initial := loc[1]
if i < len(all)-1 {
end := all[i+1][0] - 1
info = output[initial:end]
} else {
// if this is the last match
info = output[initial:]
}
outputs[file] = info
}
return outputs
}
func (e *Extractor) parseInfo(info string) map[string][]string {
tags := map[string][]string{}
reader := strings.NewReader(info)
scanner := bufio.NewScanner(reader)
lastTag := ""
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
match := tagsRx.FindStringSubmatch(line)
if len(match) > 0 {
tagName := strings.TrimSpace(strings.ToLower(match[1]))
if tagName != "" {
tagValue := strings.TrimSpace(match[2])
tags[tagName] = append(tags[tagName], tagValue)
lastTag = tagName
continue
}
}
if lastTag != "" {
match = continuationRx.FindStringSubmatch(line)
if len(match) > 0 {
if tags[lastTag] == nil {
tags[lastTag] = []string{""}
}
tagValue := tags[lastTag][0]
tags[lastTag][0] = tagValue + "\n" + strings.TrimSpace(match[1])
continue
}
}
lastTag = ""
match = coverRx.FindStringSubmatch(line)
if len(match) > 0 {
tags["has_picture"] = []string{"true"}
continue
}
match = durationRx.FindStringSubmatch(line)
if len(match) > 0 {
tags["duration"] = []string{e.parseDuration(match[1])}
if len(match) > 1 {
tags["bitrate"] = []string{match[2]}
}
continue
}
match = bitRateRx.FindStringSubmatch(line)
if len(match) > 0 {
tags["bitrate"] = []string{match[1]}
}
match = audioStreamRx.FindStringSubmatch(line)
if len(match) > 0 {
tags["samplerate"] = []string{match[2]}
tags["channels"] = []string{e.parseChannels(match[3])}
}
}
comment := tags["comment"]
if len(comment) > 0 && comment[0] == "Cover (front)" {
delete(tags, "comment")
}
return tags
}
var zeroTime = time.Date(0000, time.January, 1, 0, 0, 0, 0, time.UTC)
func (e *Extractor) parseDuration(tag string) string {
d, err := time.Parse("15:04:05", tag)
if err != nil {
return "0"
}
return strconv.FormatFloat(d.Sub(zeroTime).Seconds(), 'f', 2, 32)
}
func (e *Extractor) parseChannels(tag string) string {
switch tag {
case "mono":
return "1"
case "stereo":
return "2"
case "5.1":
return "6"
case "7.1":
return "8"
default:
return "0"
}
}
// Inputs will always be absolute paths
func init() {
metadata_old.RegisterExtractor(ExtractorID, &Extractor{ffmpeg: ffmpeg.New()})
}

View file

@ -0,0 +1,17 @@
package ffmpeg
import (
"testing"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestFFMpeg(t *testing.T) {
tests.Init(t, true)
log.SetLevel(log.LevelFatal)
RegisterFailHandler(Fail)
RunSpecs(t, "FFMpeg Suite")
}

View file

@ -0,0 +1,375 @@
package ffmpeg
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Extractor", func() {
var e *Extractor
BeforeEach(func() {
e = &Extractor{}
})
Context("extractMetadata", func() {
It("extracts MusicBrainz custom tags", func() {
const output = `
Input #0, ape, from './Capture/02 01 - Symphony No. 5 in C minor, Op. 67 I. Allegro con brio - Ludwig van Beethoven.ape':
Metadata:
ALBUM : Forever Classics
ARTIST : Ludwig van Beethoven
TITLE : Symphony No. 5 in C minor, Op. 67: I. Allegro con brio
MUSICBRAINZ_ALBUMSTATUS: official
MUSICBRAINZ_ALBUMTYPE: album
MusicBrainz_AlbumComment: MP3
Musicbrainz_Albumid: 71eb5e4a-90e2-4a31-a2d1-a96485fcb667
musicbrainz_trackid: ffe06940-727a-415a-b608-b7e45737f9d8
Musicbrainz_Artistid: 1f9df192-a621-4f54-8850-2c5373b7eac9
Musicbrainz_Albumartistid: 89ad4ac3-39f7-470e-963a-56509c546377
Musicbrainz_Releasegroupid: 708b1ae1-2d3d-34c7-b764-2732b154f5b6
musicbrainz_releasetrackid: 6fee2e35-3049-358f-83be-43b36141028b
CatalogNumber : PLD 1201
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(SatisfyAll(
HaveKeyWithValue("catalognumber", []string{"PLD 1201"}),
HaveKeyWithValue("musicbrainz_trackid", []string{"ffe06940-727a-415a-b608-b7e45737f9d8"}),
HaveKeyWithValue("musicbrainz_albumid", []string{"71eb5e4a-90e2-4a31-a2d1-a96485fcb667"}),
HaveKeyWithValue("musicbrainz_artistid", []string{"1f9df192-a621-4f54-8850-2c5373b7eac9"}),
HaveKeyWithValue("musicbrainz_albumartistid", []string{"89ad4ac3-39f7-470e-963a-56509c546377"}),
HaveKeyWithValue("musicbrainz_albumtype", []string{"album"}),
HaveKeyWithValue("musicbrainz_albumcomment", []string{"MP3"}),
))
})
It("detects embedded cover art correctly", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.mp3':
Metadata:
compilation : 1
Duration: 00:00:01.02, start: 0.000000, bitrate: 477 kb/s
Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s
Stream #0:1: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown), 600x600 [SAR 1:1 DAR 1:1], 90k tbr, 90k tbn, 90k tbc`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("has_picture", []string{"true"}))
})
It("detects embedded cover art in ffmpeg 4.4 output", func() {
const output = `
Input #0, flac, from '/run/media/naomi/Archivio/Musica/Katy Perry/Chained to the Rhythm/01 Katy Perry featuring Skip Marley - Chained to the Rhythm.flac':
Metadata:
ARTIST : Katy Perry featuring Skip Marley
Duration: 00:03:57.91, start: 0.000000, bitrate: 983 kb/s
Stream #0:0: Audio: flac, 44100 Hz, stereo, s16
Stream #0:1: Video: mjpeg (Baseline), yuvj444p(pc, bt470bg/unknown/unknown), 599x518, 90k tbr, 90k tbn, 90k tbc (attached pic)
Metadata:
comment : Cover (front)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("has_picture", []string{"true"}))
})
It("detects embedded cover art in ogg containers", func() {
const output = `
Input #0, ogg, from '/Users/deluan/Music/iTunes/iTunes Media/Music/_Testes/Jamaican In New York/01-02 Jamaican In New York (Album Version).opus':
Duration: 00:04:28.69, start: 0.007500, bitrate: 139 kb/s
Stream #0:0(eng): Audio: opus, 48000 Hz, stereo, fltp
Metadata:
ALBUM : Jamaican In New York
metadata_block_picture: AAAAAwAAAAppbWFnZS9qcGVnAAAAAAAAAAAAAAAAAAAAAAAAAAAAA4Id/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQ
TITLE : Jamaican In New York (Album Version)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKey("metadata_block_picture"))
md = md.Map(e.CustomMappings())
Expect(md).To(HaveKey("has_picture"))
})
It("detects embedded cover art in m4a containers", func() {
const output = `
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'Putumayo Presents_ Euro Groove/01 Destins et Désirs.m4a':
Metadata:
album : Putumayo Presents: Euro Groove
Duration: 00:05:15.81, start: 0.047889, bitrate: 133 kb/s
Stream #0:0[0x1](und): Audio: aac (LC) (mp4a / 0x6134706D), 44100 Hz, stereo, fltp, 125 kb/s (default)
Metadata:
creation_time : 2008-03-11T21:03:23.000000Z
vendor_id : [0][0][0][0]
Stream #0:1[0x0]: Video: png, rgb24(pc), 350x350, 90k tbr, 90k tbn (attached pic)
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("has_picture", []string{"true"}))
})
It("gets bitrate from the stream, if available", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.mp3':
Duration: 00:00:01.02, start: 0.000000, bitrate: 477 kb/s
Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("bitrate", []string{"192"}))
})
It("parses duration with milliseconds", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.mp3':
Duration: 00:05:02.63, start: 0.000000, bitrate: 140 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("duration", []string{"302.63"}))
})
It("parse flac bitrates", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.mp3':
Duration: 00:00:01.02, start: 0.000000, bitrate: 477 kb/s
Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("channels", []string{"2"}))
})
It("parse channels from the stream with bitrate", func() {
const output = `
Input #0, flac, from '/Users/deluan/Music/Music/Media/__/Crazy For You/01-01 Crazy For You.flac':
Metadata:
TITLE : Crazy For You
Duration: 00:04:13.00, start: 0.000000, bitrate: 852 kb/s
Stream #0:0: Audio: flac, 44100 Hz, stereo, s16
Stream #0:1: Video: mjpeg (Progressive), yuvj444p(pc, bt470bg/unknown/unknown), 600x600, 90k tbr, 90k tbn, 90k tbc (attached pic)
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("bitrate", []string{"852"}))
})
It("parse 7.1 channels from the stream", func() {
const output = `
Input #0, wav, from '/Users/deluan/Music/Music/Media/_/multichannel/Nums_7dot1_24_48000.wav':
Duration: 00:00:09.05, bitrate: 9216 kb/s
Stream #0:0: Audio: pcm_s24le ([1][0][0][0] / 0x0001), 48000 Hz, 7.1, s32 (24 bit), 9216 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("channels", []string{"8"}))
})
It("parse channels from the stream without bitrate", func() {
const output = `
Input #0, flac, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.flac':
Duration: 00:00:01.02, start: 0.000000, bitrate: 1371 kb/s
Stream #0:0: Audio: flac, 44100 Hz, stereo, fltp, s32 (24 bit)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("channels", []string{"2"}))
})
It("parse channels from the stream with lang", func() {
const output = `
Input #0, flac, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.m4a':
Duration: 00:00:01.02, start: 0.000000, bitrate: 1371 kb/s
Stream #0:0(eng): Audio: aac (LC) (mp4a / 0x6134706D), 44100 Hz, stereo, fltp, 262 kb/s (default)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("channels", []string{"2"}))
})
It("parse channels from the stream with lang 2", func() {
const output = `
Input #0, flac, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.m4a':
Duration: 00:00:01.02, start: 0.000000, bitrate: 1371 kb/s
Stream #0:0(eng): Audio: vorbis, 44100 Hz, stereo, fltp, 192 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("channels", []string{"2"}))
})
It("parse sampleRate from the stream", func() {
const output = `
Input #0, dsf, from '/Users/deluan/Downloads/06-04 Perpetual Change.dsf':
Duration: 00:14:19.46, start: 0.000000, bitrate: 5644 kb/s
Stream #0:0: Audio: dsd_lsbf_planar, 352800 Hz, stereo, fltp, 5644 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("samplerate", []string{"352800"}))
})
It("parse sampleRate from the stream", func() {
const output = `
Input #0, wav, from '/Users/deluan/Music/Music/Media/_/multichannel/Nums_7dot1_24_48000.wav':
Duration: 00:00:09.05, bitrate: 9216 kb/s
Stream #0:0: Audio: pcm_s24le ([1][0][0][0] / 0x0001), 48000 Hz, 7.1, s32 (24 bit), 9216 kb/s`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("samplerate", []string{"48000"}))
})
It("parses stream level tags", func() {
const output = `
Input #0, ogg, from './01-02 Drive (Teku).opus':
Metadata:
ALBUM : Hot Wheels Acceleracers Soundtrack
Duration: 00:03:37.37, start: 0.007500, bitrate: 135 kb/s
Stream #0:0(eng): Audio: opus, 48000 Hz, stereo, fltp
Metadata:
TITLE : Drive (Teku)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("title", []string{"Drive (Teku)"}))
})
It("does not overlap top level tags with the stream level tags", func() {
const output = `
Input #0, mp3, from 'groovin.mp3':
Metadata:
title : Groovin' (feat. Daniel Sneijers, Susanne Alt)
Duration: 00:03:34.28, start: 0.025056, bitrate: 323 kb/s
Metadata:
title : garbage`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("title", []string{"Groovin' (feat. Daniel Sneijers, Susanne Alt)", "garbage"}))
})
It("parses multiline tags", func() {
const outputWithMultilineComment = `
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'modulo.m4a':
Metadata:
comment : https://www.mixcloud.com/codigorock/30-minutos-com-saara-saara/
:
: Tracklist:
:
: 01. Saara Saara
: 02. Carta Corrente
: 03. X
: 04. Eclipse Lunar
: 05. Vírus de Sírius
: 06. Doktor Fritz
: 07. Wunderbar
: 08. Quarta Dimensão
Duration: 00:26:46.96, start: 0.052971, bitrate: 69 kb/s`
const expectedComment = `https://www.mixcloud.com/codigorock/30-minutos-com-saara-saara/
Tracklist:
01. Saara Saara
02. Carta Corrente
03. X
04. Eclipse Lunar
05. Vírus de Sírius
06. Doktor Fritz
07. Wunderbar
08. Quarta Dimensão`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", outputWithMultilineComment)
Expect(md).To(HaveKeyWithValue("comment", []string{expectedComment}))
})
It("parses sort tags correctly", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Downloads/椎名林檎 - 加爾基 精液 栗ノ花 - 2003/02 - ドツペルゲンガー.mp3':
Metadata:
title-sort : Dopperugengā
album : 加爾基 精液 栗ノ花
artist : 椎名林檎
album_artist : 椎名林檎
title : ドツペルゲンガー
albumsort : Kalk Samen Kuri No Hana
artist_sort : Shiina, Ringo
ALBUMARTISTSORT : Shiina, Ringo
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(SatisfyAll(
HaveKeyWithValue("title", []string{"ドツペルゲンガー"}),
HaveKeyWithValue("album", []string{"加爾基 精液 栗ノ花"}),
HaveKeyWithValue("artist", []string{"椎名林檎"}),
HaveKeyWithValue("album_artist", []string{"椎名林檎"}),
HaveKeyWithValue("title-sort", []string{"Dopperugengā"}),
HaveKeyWithValue("albumsort", []string{"Kalk Samen Kuri No Hana"}),
HaveKeyWithValue("artist_sort", []string{"Shiina, Ringo"}),
HaveKeyWithValue("albumartistsort", []string{"Shiina, Ringo"}),
))
})
It("ignores cover comment", func() {
const output = `
Input #0, mp3, from './Edie Brickell/Picture Perfect Morning/01-01 Tomorrow Comes.mp3':
Metadata:
title : Tomorrow Comes
artist : Edie Brickell
Duration: 00:03:56.12, start: 0.000000, bitrate: 332 kb/s
Stream #0:0: Audio: mp3, 44100 Hz, stereo, s16p, 320 kb/s
Stream #0:1: Video: mjpeg, yuvj420p(pc, bt470bg/unknown/unknown), 1200x1200 [SAR 72:72 DAR 1:1], 90k tbr, 90k tbn, 90k tbc
Metadata:
comment : Cover (front)`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).ToNot(HaveKey("comment"))
})
It("parses tags with spaces in the name", func() {
const output = `
Input #0, mp3, from '/Users/deluan/Music/Music/Media/_/Wyclef Jean - From the Hut, to the Projects, to the Mansion/10 - The Struggle (interlude).mp3':
Metadata:
ALBUM ARTIST : Wyclef Jean
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("album artist", []string{"Wyclef Jean"}))
})
})
It("parses an integer TBPM tag", func() {
const output = `
Input #0, mp3, from 'tests/fixtures/test.mp3':
Metadata:
TBPM : 123`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("tbpm", []string{"123"}))
})
It("parses and rounds a floating point fBPM tag", func() {
const output = `
Input #0, ogg, from 'tests/fixtures/test.ogg':
Metadata:
FBPM : 141.7`
md, _ := e.extractMetadata("tests/fixtures/test.ogg", output)
Expect(md).To(HaveKeyWithValue("fbpm", []string{"141.7"}))
})
It("parses replaygain data correctly", func() {
const output = `
Input #0, mp3, from 'test.mp3':
Metadata:
REPLAYGAIN_ALBUM_PEAK: 0.9125
REPLAYGAIN_TRACK_PEAK: 0.4512
REPLAYGAIN_TRACK_GAIN: -1.48 dB
REPLAYGAIN_ALBUM_GAIN: +3.21518 dB
Side data:
replaygain: track gain - -1.480000, track peak - 0.000011, album gain - 3.215180, album peak - 0.000021,
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(SatisfyAll(
HaveKeyWithValue("replaygain_track_gain", []string{"-1.48 dB"}),
HaveKeyWithValue("replaygain_track_peak", []string{"0.4512"}),
HaveKeyWithValue("replaygain_album_gain", []string{"+3.21518 dB"}),
HaveKeyWithValue("replaygain_album_peak", []string{"0.9125"}),
))
})
It("parses lyrics with language code", func() {
const output = `
Input #0, mp3, from 'test.mp3':
Metadata:
lyrics-eng : [00:00.00]This is
: [00:02.50]English
lyrics-xxx : [00:00.00]This is
: [00:02.50]unspecified
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(SatisfyAll(
HaveKeyWithValue("lyrics-eng", []string{
"[00:00.00]This is\n[00:02.50]English",
}),
HaveKeyWithValue("lyrics-xxx", []string{
"[00:00.00]This is\n[00:02.50]unspecified",
}),
))
})
It("parses normal LYRICS tag", func() {
const output = `
Input #0, mp3, from 'test.mp3':
Metadata:
LYRICS : [00:00.00]This is
: [00:02.50]English
`
md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
Expect(md).To(HaveKeyWithValue("lyrics", []string{
"[00:00.00]This is\n[00:02.50]English",
}))
})
})

View file

@ -0,0 +1,408 @@
package metadata_old
import (
"encoding/json"
"fmt"
"math"
"os"
"path"
"regexp"
"strconv"
"strings"
"time"
"github.com/djherbis/times"
"github.com/google/uuid"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
)
type Extractor interface {
Parse(files ...string) (map[string]ParsedTags, error)
CustomMappings() ParsedTags
Version() string
}
var extractors = map[string]Extractor{}
func RegisterExtractor(id string, parser Extractor) {
extractors[id] = parser
}
func LogExtractors() {
for id, p := range extractors {
log.Debug("Registered metadata extractor", "id", id, "version", p.Version())
}
}
func Extract(files ...string) (map[string]Tags, error) {
p, ok := extractors[conf.Server.Scanner.Extractor]
if !ok {
log.Warn("Invalid 'Scanner.Extractor' option. Using default", "requested", conf.Server.Scanner.Extractor,
"validOptions", "ffmpeg,taglib", "default", consts.DefaultScannerExtractor)
p = extractors[consts.DefaultScannerExtractor]
}
extractedTags, err := p.Parse(files...)
if err != nil {
return nil, err
}
result := map[string]Tags{}
for filePath, tags := range extractedTags {
fileInfo, err := os.Stat(filePath)
if err != nil {
log.Warn("Error stating file. Skipping", "filePath", filePath, err)
continue
}
tags = tags.Map(p.CustomMappings())
result[filePath] = NewTag(filePath, fileInfo, tags)
}
return result, nil
}
func NewTag(filePath string, fileInfo os.FileInfo, tags ParsedTags) Tags {
for t, values := range tags {
values = removeDuplicatesAndEmpty(values)
if len(values) == 0 {
delete(tags, t)
continue
}
tags[t] = values
}
return Tags{
filePath: filePath,
fileInfo: fileInfo,
Tags: tags,
}
}
func removeDuplicatesAndEmpty(values []string) []string {
encountered := map[string]struct{}{}
empty := true
result := make([]string, 0, len(values))
for _, v := range values {
if _, ok := encountered[v]; ok {
continue
}
encountered[v] = struct{}{}
empty = empty && v == ""
result = append(result, v)
}
if empty {
return nil
}
return result
}
type ParsedTags map[string][]string
func (p ParsedTags) Map(customMappings ParsedTags) ParsedTags {
if customMappings == nil {
return p
}
for tagName, alternatives := range customMappings {
for _, altName := range alternatives {
if altValue, ok := p[altName]; ok {
p[tagName] = append(p[tagName], altValue...)
delete(p, altName)
}
}
}
return p
}
type Tags struct {
filePath string
fileInfo os.FileInfo
Tags ParsedTags
}
// Common tags
func (t Tags) Title() string { return t.getFirstTagValue("title", "sort_name", "titlesort") }
func (t Tags) Album() string { return t.getFirstTagValue("album", "sort_album", "albumsort") }
func (t Tags) Artist() string { return t.getFirstTagValue("artist", "sort_artist", "artistsort") }
func (t Tags) AlbumArtist() string {
return t.getFirstTagValue("album_artist", "album artist", "albumartist")
}
func (t Tags) SortTitle() string { return t.getSortTag("tsot", "title", "name") }
func (t Tags) SortAlbum() string { return t.getSortTag("tsoa", "album") }
func (t Tags) SortArtist() string { return t.getSortTag("tsop", "artist") }
func (t Tags) SortAlbumArtist() string { return t.getSortTag("tso2", "albumartist", "album_artist") }
func (t Tags) Genres() []string { return t.getAllTagValues("genre") }
func (t Tags) Date() (int, string) { return t.getDate("date") }
func (t Tags) OriginalDate() (int, string) { return t.getDate("originaldate") }
func (t Tags) ReleaseDate() (int, string) { return t.getDate("releasedate") }
func (t Tags) Comment() string { return t.getFirstTagValue("comment") }
func (t Tags) Compilation() bool { return t.getBool("tcmp", "compilation", "wm/iscompilation") }
func (t Tags) TrackNumber() (int, int) { return t.getTuple("track", "tracknumber") }
func (t Tags) DiscNumber() (int, int) { return t.getTuple("disc", "discnumber") }
func (t Tags) DiscSubtitle() string {
return t.getFirstTagValue("tsst", "discsubtitle", "setsubtitle")
}
func (t Tags) CatalogNum() string { return t.getFirstTagValue("catalognumber") }
func (t Tags) Bpm() int { return (int)(math.Round(t.getFloat("tbpm", "bpm", "fbpm"))) }
func (t Tags) HasPicture() bool { return t.getFirstTagValue("has_picture") != "" }
// MusicBrainz Identifiers
func (t Tags) MbzReleaseTrackID() string {
return t.getMbzID("musicbrainz_releasetrackid", "musicbrainz release track id")
}
func (t Tags) MbzRecordingID() string {
return t.getMbzID("musicbrainz_trackid", "musicbrainz track id")
}
func (t Tags) MbzAlbumID() string { return t.getMbzID("musicbrainz_albumid", "musicbrainz album id") }
func (t Tags) MbzArtistID() string {
return t.getMbzID("musicbrainz_artistid", "musicbrainz artist id")
}
func (t Tags) MbzAlbumArtistID() string {
return t.getMbzID("musicbrainz_albumartistid", "musicbrainz album artist id")
}
func (t Tags) MbzAlbumType() string {
return t.getFirstTagValue("musicbrainz_albumtype", "musicbrainz album type")
}
func (t Tags) MbzAlbumComment() string {
return t.getFirstTagValue("musicbrainz_albumcomment", "musicbrainz album comment")
}
// Gain Properties
func (t Tags) RGAlbumGain() float64 {
return t.getGainValue("replaygain_album_gain", "r128_album_gain")
}
func (t Tags) RGAlbumPeak() float64 { return t.getPeakValue("replaygain_album_peak") }
func (t Tags) RGTrackGain() float64 {
return t.getGainValue("replaygain_track_gain", "r128_track_gain")
}
func (t Tags) RGTrackPeak() float64 { return t.getPeakValue("replaygain_track_peak") }
// File properties
func (t Tags) Duration() float32 { return float32(t.getFloat("duration")) }
func (t Tags) SampleRate() int { return t.getInt("samplerate") }
func (t Tags) BitRate() int { return t.getInt("bitrate") }
func (t Tags) Channels() int { return t.getInt("channels") }
func (t Tags) ModificationTime() time.Time { return t.fileInfo.ModTime() }
func (t Tags) Size() int64 { return t.fileInfo.Size() }
func (t Tags) FilePath() string { return t.filePath }
func (t Tags) Suffix() string { return strings.ToLower(strings.TrimPrefix(path.Ext(t.filePath), ".")) }
func (t Tags) BirthTime() time.Time {
if ts := times.Get(t.fileInfo); ts.HasBirthTime() {
return ts.BirthTime()
}
return time.Now()
}
func (t Tags) Lyrics() string {
lyricList := model.LyricList{}
basicLyrics := t.getAllTagValues("lyrics", "unsynced_lyrics", "unsynced lyrics", "unsyncedlyrics")
for _, value := range basicLyrics {
lyrics, err := model.ToLyrics("xxx", value)
if err != nil {
log.Warn("Unexpected failure occurred when parsing lyrics", "file", t.filePath, "error", err)
continue
}
lyricList = append(lyricList, *lyrics)
}
for tag, value := range t.Tags {
if strings.HasPrefix(tag, "lyrics-") {
language := strings.TrimSpace(strings.TrimPrefix(tag, "lyrics-"))
if language == "" {
language = "xxx"
}
for _, text := range value {
lyrics, err := model.ToLyrics(language, text)
if err != nil {
log.Warn("Unexpected failure occurred when parsing lyrics", "file", t.filePath, "error", err)
continue
}
lyricList = append(lyricList, *lyrics)
}
}
}
res, err := json.Marshal(lyricList)
if err != nil {
log.Warn("Unexpected error occurred when serializing lyrics", "file", t.filePath, "error", err)
return ""
}
return string(res)
}
func (t Tags) getGainValue(rgTagName, r128TagName string) float64 {
// Check for ReplayGain first
// ReplayGain is in the form [-]a.bb dB and normalized to -18dB
var tag = t.getFirstTagValue(rgTagName)
if tag != "" {
tag = strings.TrimSpace(strings.Replace(tag, "dB", "", 1))
var value, err = strconv.ParseFloat(tag, 64)
if err != nil || value == math.Inf(-1) || value == math.Inf(1) {
return 0
}
return value
}
// If ReplayGain is not found, check for R128 gain
// R128 gain is a Q7.8 fixed point number normalized to -23dB
tag = t.getFirstTagValue(r128TagName)
if tag != "" {
var iValue, err = strconv.Atoi(tag)
if err != nil {
return 0
}
// Convert Q7.8 to float
var value = float64(iValue) / 256.0
// Adding 5 dB to normalize with ReplayGain level
return value + 5
}
return 0
}
func (t Tags) getPeakValue(tagName string) float64 {
var tag = t.getFirstTagValue(tagName)
var value, err = strconv.ParseFloat(tag, 64)
if err != nil || value == math.Inf(-1) || value == math.Inf(1) {
// A default of 1 for peak value results in no changes
return 1
}
return value
}
func (t Tags) getTags(tagNames ...string) []string {
for _, tag := range tagNames {
if v, ok := t.Tags[tag]; ok {
return v
}
}
return nil
}
func (t Tags) getFirstTagValue(tagNames ...string) string {
ts := t.getTags(tagNames...)
if len(ts) > 0 {
return ts[0]
}
return ""
}
func (t Tags) getAllTagValues(tagNames ...string) []string {
values := make([]string, 0, len(tagNames)*2)
for _, tag := range tagNames {
if v, ok := t.Tags[tag]; ok {
values = append(values, v...)
}
}
return values
}
func (t Tags) getSortTag(originalTag string, tagNames ...string) string {
formats := []string{"sort%s", "sort_%s", "sort-%s", "%ssort", "%s_sort", "%s-sort"}
all := make([]string, 1, len(tagNames)*len(formats)+1)
all[0] = originalTag
for _, tag := range tagNames {
for _, format := range formats {
name := fmt.Sprintf(format, tag)
all = append(all, name)
}
}
return t.getFirstTagValue(all...)
}
var dateRegex = regexp.MustCompile(`([12]\d\d\d)`)
func (t Tags) getDate(tagNames ...string) (int, string) {
tag := t.getFirstTagValue(tagNames...)
if len(tag) < 4 {
return 0, ""
}
// first get just the year
match := dateRegex.FindStringSubmatch(tag)
if len(match) == 0 {
log.Warn("Error parsing "+tagNames[0]+" field for year", "file", t.filePath, "date", tag)
return 0, ""
}
year, _ := strconv.Atoi(match[1])
if len(tag) < 5 {
return year, match[1]
}
//then try YYYY-MM-DD
if len(tag) > 10 {
tag = tag[:10]
}
layout := "2006-01-02"
_, err := time.Parse(layout, tag)
if err != nil {
layout = "2006-01"
_, err = time.Parse(layout, tag)
if err != nil {
log.Warn("Error parsing "+tagNames[0]+" field for month + day", "file", t.filePath, "date", tag)
return year, match[1]
}
}
return year, tag
}
func (t Tags) getBool(tagNames ...string) bool {
tag := t.getFirstTagValue(tagNames...)
if tag == "" {
return false
}
i, _ := strconv.Atoi(strings.TrimSpace(tag))
return i == 1
}
func (t Tags) getTuple(tagNames ...string) (int, int) {
tag := t.getFirstTagValue(tagNames...)
if tag == "" {
return 0, 0
}
tuple := strings.Split(tag, "/")
t1, t2 := 0, 0
t1, _ = strconv.Atoi(tuple[0])
if len(tuple) > 1 {
t2, _ = strconv.Atoi(tuple[1])
} else {
t2tag := t.getFirstTagValue(tagNames[0] + "total")
t2, _ = strconv.Atoi(t2tag)
}
return t1, t2
}
func (t Tags) getMbzID(tagNames ...string) string {
tag := t.getFirstTagValue(tagNames...)
if _, err := uuid.Parse(tag); err != nil {
return ""
}
return tag
}
func (t Tags) getInt(tagNames ...string) int {
tag := t.getFirstTagValue(tagNames...)
i, _ := strconv.Atoi(tag)
return i
}
func (t Tags) getFloat(tagNames ...string) float64 {
var tag = t.getFirstTagValue(tagNames...)
var value, err = strconv.ParseFloat(tag, 64)
if err != nil {
return 0
}
return value
}

View file

@ -0,0 +1,144 @@
package metadata_old
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Tags", func() {
DescribeTable("getDate",
func(tag string, expectedYear int, expectedDate string) {
md := &Tags{}
md.Tags = map[string][]string{"date": {tag}}
testYear, testDate := md.Date()
Expect(testYear).To(Equal(expectedYear))
Expect(testDate).To(Equal(expectedDate))
},
Entry(nil, "1985", 1985, "1985"),
Entry(nil, "2002-01", 2002, "2002-01"),
Entry(nil, "1969.06", 1969, "1969"),
Entry(nil, "1980.07.25", 1980, "1980"),
Entry(nil, "2004-00-00", 2004, "2004"),
Entry(nil, "2016-12-31", 2016, "2016-12-31"),
Entry(nil, "2013-May-12", 2013, "2013"),
Entry(nil, "May 12, 2016", 2016, "2016"),
Entry(nil, "01/10/1990", 1990, "1990"),
Entry(nil, "invalid", 0, ""),
)
Describe("getMbzID", func() {
It("return a valid MBID", func() {
md := &Tags{}
md.Tags = map[string][]string{
"musicbrainz_trackid": {"8f84da07-09a0-477b-b216-cc982dabcde1"},
"musicbrainz_releasetrackid": {"6caf16d3-0b20-3fe6-8020-52e31831bc11"},
"musicbrainz_albumid": {"f68c985d-f18b-4f4a-b7f0-87837cf3fbf9"},
"musicbrainz_artistid": {"89ad4ac3-39f7-470e-963a-56509c546377"},
"musicbrainz_albumartistid": {"ada7a83c-e3e1-40f1-93f9-3e73dbc9298a"},
}
Expect(md.MbzRecordingID()).To(Equal("8f84da07-09a0-477b-b216-cc982dabcde1"))
Expect(md.MbzReleaseTrackID()).To(Equal("6caf16d3-0b20-3fe6-8020-52e31831bc11"))
Expect(md.MbzAlbumID()).To(Equal("f68c985d-f18b-4f4a-b7f0-87837cf3fbf9"))
Expect(md.MbzArtistID()).To(Equal("89ad4ac3-39f7-470e-963a-56509c546377"))
Expect(md.MbzAlbumArtistID()).To(Equal("ada7a83c-e3e1-40f1-93f9-3e73dbc9298a"))
})
It("return empty string for invalid MBID", func() {
md := &Tags{}
md.Tags = map[string][]string{
"musicbrainz_trackid": {"11406732-6"},
"musicbrainz_albumid": {"11406732"},
"musicbrainz_artistid": {"200455"},
"musicbrainz_albumartistid": {"194"},
}
Expect(md.MbzRecordingID()).To(Equal(""))
Expect(md.MbzAlbumID()).To(Equal(""))
Expect(md.MbzArtistID()).To(Equal(""))
Expect(md.MbzAlbumArtistID()).To(Equal(""))
})
})
Describe("getAllTagValues", func() {
It("returns values from all tag names", func() {
md := &Tags{}
md.Tags = map[string][]string{
"genre": {"Rock", "Pop", "New Wave"},
}
Expect(md.Genres()).To(ConsistOf("Rock", "Pop", "New Wave"))
})
})
Describe("removeDuplicatesAndEmpty", func() {
It("removes duplicates", func() {
md := NewTag("/music/artist/album01/Song.mp3", nil, ParsedTags{
"genre": []string{"pop", "rock", "pop"},
"date": []string{"2023-03-01", "2023-03-01"},
"mood": []string{"happy", "sad"},
})
Expect(md.Tags).To(HaveKeyWithValue("genre", []string{"pop", "rock"}))
Expect(md.Tags).To(HaveKeyWithValue("date", []string{"2023-03-01"}))
Expect(md.Tags).To(HaveKeyWithValue("mood", []string{"happy", "sad"}))
})
It("removes empty tags", func() {
md := NewTag("/music/artist/album01/Song.mp3", nil, ParsedTags{
"genre": []string{"pop", "rock", "pop"},
"mood": []string{"", ""},
})
Expect(md.Tags).To(HaveKeyWithValue("genre", []string{"pop", "rock"}))
Expect(md.Tags).ToNot(HaveKey("mood"))
})
})
Describe("BPM", func() {
var t *Tags
BeforeEach(func() {
t = &Tags{Tags: map[string][]string{
"fbpm": []string{"141.7"},
}}
})
It("rounds a floating point fBPM tag", func() {
Expect(t.Bpm()).To(Equal(142))
})
})
Describe("ReplayGain", func() {
DescribeTable("getGainValue",
func(tag string, expected float64) {
md := &Tags{}
md.Tags = map[string][]string{"replaygain_track_gain": {tag}}
Expect(md.RGTrackGain()).To(Equal(expected))
},
Entry("0", "0", 0.0),
Entry("1.2dB", "1.2dB", 1.2),
Entry("Infinity", "Infinity", 0.0),
Entry("Invalid value", "INVALID VALUE", 0.0),
)
DescribeTable("getPeakValue",
func(tag string, expected float64) {
md := &Tags{}
md.Tags = map[string][]string{"replaygain_track_peak": {tag}}
Expect(md.RGTrackPeak()).To(Equal(expected))
},
Entry("0", "0", 0.0),
Entry("0.5", "0.5", 0.5),
Entry("Invalid dB suffix", "0.7dB", 1.0),
Entry("Infinity", "Infinity", 1.0),
Entry("Invalid value", "INVALID VALUE", 1.0),
)
DescribeTable("getR128GainValue",
func(tag string, expected float64) {
md := &Tags{}
md.Tags = map[string][]string{"r128_track_gain": {tag}}
Expect(md.RGTrackGain()).To(Equal(expected))
},
Entry("0", "0", 5.0),
Entry("-3776", "-3776", -9.75),
Entry("Infinity", "Infinity", 0.0),
Entry("Invalid value", "INVALID VALUE", 0.0),
)
})
})

View file

@ -0,0 +1,17 @@
package metadata_old
import (
"testing"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMetadata(t *testing.T) {
tests.Init(t, true)
log.SetLevel(log.LevelFatal)
RegisterFailHandler(Fail)
RunSpecs(t, "Metadata Suite")
}

View file

@ -0,0 +1,95 @@
package metadata_old_test
import (
"cmp"
"encoding/json"
"slices"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/core/ffmpeg"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner/metadata_old"
_ "github.com/navidrome/navidrome/scanner/metadata_old/ffmpeg"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Tags", func() {
var zero int64 = 0
var secondTs int64 = 2500
makeLyrics := func(synced bool, lang, secondLine string) model.Lyrics {
lines := []model.Line{
{Value: "This is"},
{Value: secondLine},
}
if synced {
lines[0].Start = &zero
lines[1].Start = &secondTs
}
lyrics := model.Lyrics{
Lang: lang,
Line: lines,
Synced: synced,
}
return lyrics
}
sortLyrics := func(lines model.LyricList) model.LyricList {
slices.SortFunc(lines, func(a, b model.Lyrics) int {
langDiff := cmp.Compare(a.Lang, b.Lang)
if langDiff != 0 {
return langDiff
}
return cmp.Compare(a.Line[1].Value, b.Line[1].Value)
})
return lines
}
compareLyrics := func(m metadata_old.Tags, expected model.LyricList) {
lyrics := model.LyricList{}
Expect(json.Unmarshal([]byte(m.Lyrics()), &lyrics)).To(BeNil())
Expect(sortLyrics(lyrics)).To(Equal(sortLyrics(expected)))
}
// Only run these tests if FFmpeg is available
FFmpegContext := XContext
if ffmpeg.New().IsAvailable() {
FFmpegContext = Context
}
FFmpegContext("Extract with FFmpeg", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
conf.Server.Scanner.Extractor = "ffmpeg"
})
DescribeTable("Lyrics test",
func(file string) {
path := "tests/fixtures/" + file
mds, err := metadata_old.Extract(path)
Expect(err).ToNot(HaveOccurred())
Expect(mds).To(HaveLen(1))
m := mds[path]
compareLyrics(m, model.LyricList{
makeLyrics(true, "eng", "English"),
makeLyrics(true, "xxx", "unspecified"),
})
},
Entry("Parses AIFF file", "test.aiff"),
Entry("Parses MP3 files", "test.mp3"),
// Disabled, because it fails in pipeline
// Entry("Parses WAV files", "test.wav"),
// FFMPEG behaves very weirdly for multivalued tags for non-ID3
// Specifically, they are separated by ";, which is indistinguishable
// from other fields
)
})
})