From 8e2052ff95d5695a79f139793ae9b2bfeefe03f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Tue, 17 Dec 2024 17:10:55 -0500 Subject: [PATCH] feat(Insights): add anonymous usage data collection (#3543) * feat(insights): initial code (WIP) * feat(insights): add more info * feat(insights): add fs info * feat(insights): export insights.Data Signed-off-by: Deluan * feat(insights): more config info Signed-off-by: Deluan * refactor(insights): move data struct to its own package Signed-off-by: Deluan * refactor(insights): omit some attrs if empty Signed-off-by: Deluan * feat(insights): send insights to server, add option to disable Signed-off-by: Deluan * fix(insights): remove info about anonymous login Signed-off-by: Deluan * chore(insights): fix lint Signed-off-by: Deluan * fix(insights): disable collector if EnableExternalServices is false Signed-off-by: Deluan * fix(insights): fix type casting for 32bit platforms Signed-off-by: Deluan * fix(insights): remove EnableExternalServices from the collection (as it will always be false) Signed-off-by: Deluan * chore(insights): fix lint Signed-off-by: Deluan * refactor(insights): rename function for consistency Signed-off-by: Deluan * feat(insights): log the data sent to the collector server Signed-off-by: Deluan * feat(insights): add last collection timestamp to the "about" dialog. Also add opt-out info to the SignUp form Signed-off-by: Deluan * feat(insights): only sends the initial data collection after an admin user is created Signed-off-by: Deluan * feat(insights): remove dangling comment Signed-off-by: Deluan * feat(insights): Translate insights messages Signed-off-by: Deluan * fix(insights): reporting empty library Signed-off-by: Deluan * refactor: move URL to consts.js Signed-off-by: Deluan --------- Signed-off-by: Deluan --- Makefile | 2 +- cmd/root.go | 19 +- cmd/wire_gen.go | 48 +++-- cmd/wire_injectors.go | 7 + conf/configuration.go | 3 + consts/consts.go | 6 + core/metrics/insights.go | 227 +++++++++++++++++++++ core/metrics/insights/data.go | 71 +++++++ core/metrics/insights_darwin.go | 37 ++++ core/metrics/insights_default.go | 9 + core/metrics/insights_linux.go | 77 +++++++ core/metrics/insights_windows.go | 53 +++++ core/{metrics.go => metrics/prometheus.go} | 2 +- core/wire_providers.go | 2 + go.mod | 2 +- model/share.go | 1 + persistence/share_repository.go | 1 + resources/i18n/pt.json | 6 +- scanner/scanner.go | 5 +- server/nativeapi/native_api.go | 17 +- server/server.go | 14 +- ui/src/App.jsx | 1 + ui/src/consts.js | 3 + ui/src/dialogs/AboutDialog.jsx | 18 +- ui/src/i18n/en.json | 6 +- ui/src/layout/Login.jsx | 67 ++++++ 26 files changed, 665 insertions(+), 39 deletions(-) create mode 100644 core/metrics/insights.go create mode 100644 core/metrics/insights/data.go create mode 100644 core/metrics/insights_darwin.go create mode 100644 core/metrics/insights_default.go create mode 100644 core/metrics/insights_linux.go create mode 100644 core/metrics/insights_windows.go rename core/{metrics.go => metrics/prometheus.go} (99%) diff --git a/Makefile b/Makefile index e581bacd6..87cadc71e 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ format: ##@Development Format code .PHONY: format wire: check_go_env ##@Development Update Dependency Injection - go run github.com/google/wire/cmd/wire@latest ./... + go run github.com/google/wire/cmd/wire@latest gen -tags=netgo ./... .PHONY: wire snapshots: ##@Development Update (GoLang) Snapshot tests diff --git a/cmd/root.go b/cmd/root.go index 67407c489..d13a092ef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" - "github.com/navidrome/navidrome/core" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/resources" @@ -80,6 +80,7 @@ func runNavidrome(ctx context.Context) { g.Go(startPlaybackServer(ctx)) g.Go(schedulePeriodicScan(ctx)) g.Go(schedulePeriodicBackup(ctx)) + g.Go(startInsightsCollector(ctx)) if err := g.Wait(); err != nil { log.Error("Fatal error in Navidrome. Aborting", err) @@ -111,7 +112,7 @@ func startServer(ctx context.Context) func() error { } if conf.Server.Prometheus.Enabled { // blocking call because takes <1ms but useful if fails - core.WriteInitialMetrics() + metrics.WriteInitialMetrics() a.MountRouter("Prometheus metrics", conf.Server.Prometheus.MetricsPath, promhttp.Handler()) } if conf.Server.DevEnableProfiler { @@ -199,6 +200,20 @@ func startScheduler(ctx context.Context) func() error { } } +// startInsightsCollector starts the Navidrome Insight Collector, if configured. +func startInsightsCollector(ctx context.Context) func() error { + return func() error { + if !conf.Server.EnableInsightsCollector { + log.Info(ctx, "Insight Collector is DISABLED") + return nil + } + log.Info(ctx, "Starting Insight Collector") + ic := CreateInsights() + ic.Run(ctx) + return nil + } +} + // startPlaybackServer starts the Navidrome playback server, if configured. // It is responsible for the Jukebox functionality func startPlaybackServer(ctx context.Context) func() error { diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index 2915d05af..2725853d4 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire gen -tags "netgo" //go:build !wireinject // +build !wireinject @@ -14,6 +14,7 @@ import ( "github.com/navidrome/navidrome/core/agents/listenbrainz" "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/core/playback" "github.com/navidrome/navidrome/core/scrobbler" "github.com/navidrome/navidrome/db" @@ -29,25 +30,27 @@ import ( // Injectors from wire_injectors.go: func CreateServer(musicFolder string) *server.Server { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) broker := events.GetBroker() - serverServer := server.New(dataStore, broker) + insights := metrics.GetInstance(dataStore) + serverServer := server.New(dataStore, broker, insights) return serverServer } func CreateNativeAPIRouter() *nativeapi.Router { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) share := core.NewShare(dataStore) playlists := core.NewPlaylists(dataStore) - router := nativeapi.New(dataStore, share, playlists) + insights := metrics.GetInstance(dataStore) + router := nativeapi.New(dataStore, share, playlists, insights) return router } func CreateSubsonicAPIRouter() *subsonic.Router { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) fileCache := artwork.GetImageCache() fFmpeg := ffmpeg.New() agentsAgents := agents.New(dataStore) @@ -69,8 +72,8 @@ func CreateSubsonicAPIRouter() *subsonic.Router { } func CreatePublicRouter() *public.Router { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) fileCache := artwork.GetImageCache() fFmpeg := ffmpeg.New() agentsAgents := agents.New(dataStore) @@ -85,22 +88,29 @@ func CreatePublicRouter() *public.Router { } func CreateLastFMRouter() *lastfm.Router { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) router := lastfm.NewRouter(dataStore) return router } func CreateListenBrainzRouter() *listenbrainz.Router { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) router := listenbrainz.NewRouter(dataStore) return router } +func CreateInsights() metrics.Insights { + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) + insights := metrics.GetInstance(dataStore) + return insights +} + func GetScanner() scanner.Scanner { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) playlists := core.NewPlaylists(dataStore) fileCache := artwork.GetImageCache() fFmpeg := ffmpeg.New() @@ -114,8 +124,8 @@ func GetScanner() scanner.Scanner { } func GetPlaybackServer() playback.PlaybackServer { - dbDB := db.Db() - dataStore := persistence.New(dbDB) + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) playbackServer := playback.GetInstance(dataStore) return playbackServer } diff --git a/cmd/wire_injectors.go b/cmd/wire_injectors.go index 994056e37..ef58a55c7 100644 --- a/cmd/wire_injectors.go +++ b/cmd/wire_injectors.go @@ -8,6 +8,7 @@ import ( "github.com/navidrome/navidrome/core/agents/lastfm" "github.com/navidrome/navidrome/core/agents/listenbrainz" "github.com/navidrome/navidrome/core/artwork" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/core/playback" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/persistence" @@ -70,6 +71,12 @@ func CreateListenBrainzRouter() *listenbrainz.Router { )) } +func CreateInsights() metrics.Insights { + panic(wire.Build( + allProviders, + )) +} + func GetScanner() scanner.Scanner { panic(wire.Build( allProviders, diff --git a/conf/configuration.go b/conf/configuration.go index e9464af41..17e8caedf 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -42,6 +42,7 @@ type configOptions struct { EnableTranscodingConfig bool EnableDownloads bool EnableExternalServices bool + EnableInsightsCollector bool EnableMediaFileCoverArt bool TranscodingCacheSize string ImageCacheSize string @@ -295,6 +296,7 @@ func parseIniFileConfiguration() { func disableExternalServices() { log.Info("All external integrations are DISABLED!") + Server.EnableInsightsCollector = false Server.LastFM.Enabled = false Server.Spotify.ID = "" Server.ListenBrainz.Enabled = false @@ -412,6 +414,7 @@ func init() { viper.SetDefault("enablereplaygain", true) viper.SetDefault("enablecoveranimation", true) viper.SetDefault("gatrackingid", "") + viper.SetDefault("enableinsightscollector", true) viper.SetDefault("enablelogredacting", true) viper.SetDefault("authrequestlimit", 5) viper.SetDefault("authwindowlength", 20*time.Second) diff --git a/consts/consts.go b/consts/consts.go index 28dfc9427..f30964d43 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -86,6 +86,12 @@ const ( AlbumPlayCountModeNormalized = "normalized" ) +const ( + InsightsIDKey = "InsightsID" + InsightsEndpoint = "https://insights.navidrome.org/collect" + InsightsUpdateInterval = 24 * time.Hour +) + var ( DefaultDownsamplingFormat = "opus" DefaultTranscodings = []struct { diff --git a/core/metrics/insights.go b/core/metrics/insights.go new file mode 100644 index 000000000..3ac65c1fe --- /dev/null +++ b/core/metrics/insights.go @@ -0,0 +1,227 @@ +package metrics + +import ( + "bytes" + "context" + "encoding/json" + "math" + "net/http" + "path/filepath" + "runtime" + "runtime/debug" + "sync" + "time" + + "github.com/Masterminds/squirrel" + "github.com/google/uuid" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/metrics/insights" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/singleton" +) + +type Insights interface { + Run(ctx context.Context) + LastRun(ctx context.Context) (timestamp time.Time, success bool) +} + +var ( + insightsID string +) + +type insightsCollector struct { + ds model.DataStore + lastRun time.Time + lastStatus bool +} + +func GetInstance(ds model.DataStore) Insights { + return singleton.GetInstance(func() *insightsCollector { + id, err := ds.Property(context.TODO()).Get(consts.InsightsIDKey) + if err != nil { + log.Trace("Could not get Insights ID from DB. Creating one", err) + id = uuid.NewString() + err = ds.Property(context.TODO()).Put(consts.InsightsIDKey, id) + if err != nil { + log.Trace("Could not save Insights ID to DB", err) + } + } + insightsID = id + return &insightsCollector{ds: ds} + }) +} + +func (c *insightsCollector) Run(ctx context.Context) { + for { + c.sendInsights(ctx) + select { + case <-time.After(consts.InsightsUpdateInterval): + continue + case <-ctx.Done(): + return + } + } +} + +func (c *insightsCollector) LastRun(context.Context) (timestamp time.Time, success bool) { + return c.lastRun, c.lastStatus +} + +func (c *insightsCollector) sendInsights(ctx context.Context) { + count, err := c.ds.User(ctx).CountAll(model.QueryOptions{}) + if err != nil { + log.Trace(ctx, "Could not check user count", err) + return + } + if count == 0 { + log.Trace(ctx, "No users found, skipping Insights data collection") + return + } + hc := &http.Client{ + Timeout: consts.DefaultHttpClientTimeOut, + } + data := c.collect(ctx) + if data == nil { + return + } + body := bytes.NewReader(data) + req, err := http.NewRequestWithContext(ctx, "POST", consts.InsightsEndpoint, body) + if err != nil { + log.Trace(ctx, "Could not create Insights request", err) + return + } + req.Header.Set("Content-Type", "application/json") + resp, err := hc.Do(req) + if err != nil { + log.Trace(ctx, "Could not send Insights data", err) + return + } + log.Info(ctx, "Sent Insights data (for details see http://navidrome.org/docs/getting-started/insights", "data", + string(data), "server", consts.InsightsEndpoint, "status", resp.Status) + c.lastRun = time.Now() + c.lastStatus = resp.StatusCode < 300 + resp.Body.Close() +} + +func buildInfo() (map[string]string, string) { + bInfo := map[string]string{} + var version string + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Value == "" { + continue + } + bInfo[setting.Key] = setting.Value + } + version = info.GoVersion + } + return bInfo, version +} + +func getFSInfo(path string) *insights.FSInfo { + var info insights.FSInfo + + // Normalize the path + absPath, err := filepath.Abs(path) + if err != nil { + return nil + } + absPath = filepath.Clean(absPath) + + fsType, err := getFilesystemType(absPath) + if err != nil { + return nil + } + info.Type = fsType + return &info +} + +var staticData = sync.OnceValue(func() insights.Data { + // Basic info + data := insights.Data{ + InsightsID: insightsID, + Version: consts.Version, + } + + // Build info + data.Build.Settings, data.Build.GoVersion = buildInfo() + + // OS info + data.OS.Type = runtime.GOOS + data.OS.Arch = runtime.GOARCH + data.OS.NumCPU = runtime.NumCPU() + data.OS.Version, data.OS.Distro = getOSVersion() + + // FS info + data.FS.Music = getFSInfo(conf.Server.MusicFolder) + data.FS.Data = getFSInfo(conf.Server.DataFolder) + if conf.Server.CacheFolder != "" { + data.FS.Cache = getFSInfo(conf.Server.CacheFolder) + } + if conf.Server.Backup.Path != "" { + data.FS.Backup = getFSInfo(conf.Server.Backup.Path) + } + + // Config info + data.Config.LogLevel = conf.Server.LogLevel + data.Config.LogFileConfigured = conf.Server.LogFile != "" + data.Config.TLSConfigured = conf.Server.TLSCert != "" && conf.Server.TLSKey != "" + data.Config.DefaultBackgroundURLSet = conf.Server.UILoginBackgroundURL == consts.DefaultUILoginBackgroundURL + data.Config.EnableArtworkPrecache = conf.Server.EnableArtworkPrecache + data.Config.EnableCoverAnimation = conf.Server.EnableCoverAnimation + data.Config.EnableDownloads = conf.Server.EnableDownloads + data.Config.EnableSharing = conf.Server.EnableSharing + data.Config.EnableStarRating = conf.Server.EnableStarRating + data.Config.EnableLastFM = conf.Server.LastFM.Enabled + data.Config.EnableListenBrainz = conf.Server.ListenBrainz.Enabled + data.Config.EnableMediaFileCoverArt = conf.Server.EnableMediaFileCoverArt + data.Config.EnableSpotify = conf.Server.Spotify.ID != "" + data.Config.EnableJukebox = conf.Server.Jukebox.Enabled + data.Config.EnablePrometheus = conf.Server.Prometheus.Enabled + data.Config.TranscodingCacheSize = conf.Server.TranscodingCacheSize + data.Config.ImageCacheSize = conf.Server.ImageCacheSize + data.Config.ScanSchedule = conf.Server.ScanSchedule + data.Config.SessionTimeout = uint64(math.Trunc(conf.Server.SessionTimeout.Seconds())) + data.Config.SearchFullString = conf.Server.SearchFullString + data.Config.RecentlyAddedByModTime = conf.Server.RecentlyAddedByModTime + data.Config.PreferSortTags = conf.Server.PreferSortTags + data.Config.BackupSchedule = conf.Server.Backup.Schedule + data.Config.BackupCount = conf.Server.Backup.Count + data.Config.DevActivityPanel = conf.Server.DevActivityPanel + + return data +}) + +func (c *insightsCollector) collect(ctx context.Context) []byte { + data := staticData() + data.Uptime = time.Since(consts.ServerStart).Milliseconds() / 1000 + + // Library info + data.Library.Tracks, _ = c.ds.MediaFile(ctx).CountAll() + data.Library.Albums, _ = c.ds.Album(ctx).CountAll() + data.Library.Artists, _ = c.ds.Artist(ctx).CountAll() + data.Library.Playlists, _ = c.ds.Playlist(ctx).Count() + data.Library.Shares, _ = c.ds.Share(ctx).CountAll() + data.Library.Radios, _ = c.ds.Radio(ctx).Count() + data.Library.ActiveUsers, _ = c.ds.User(ctx).CountAll(model.QueryOptions{ + Filters: squirrel.Gt{"last_access_at": time.Now().Add(-7 * 24 * time.Hour)}, + }) + + // Memory info + var m runtime.MemStats + runtime.ReadMemStats(&m) + data.Mem.Alloc = m.Alloc + data.Mem.TotalAlloc = m.TotalAlloc + data.Mem.Sys = m.Sys + data.Mem.NumGC = m.NumGC + + // Marshal to JSON + resp, err := json.Marshal(data) + if err != nil { + log.Trace(ctx, "Could not marshal Insights data", err) + return nil + } + return resp +} diff --git a/core/metrics/insights/data.go b/core/metrics/insights/data.go new file mode 100644 index 000000000..9e0d1ade9 --- /dev/null +++ b/core/metrics/insights/data.go @@ -0,0 +1,71 @@ +package insights + +type Data struct { + InsightsID string `json:"id"` + Version string `json:"version"` + Uptime int64 `json:"uptime"` + Build struct { + // build settings used by the Go compiler + Settings map[string]string `json:"settings"` + GoVersion string `json:"goVersion"` + } `json:"build"` + OS struct { + Type string `json:"type"` + Distro string `json:"distro,omitempty"` + Version string `json:"version,omitempty"` + Arch string `json:"arch"` + NumCPU int `json:"numCPU"` + } `json:"os"` + Mem struct { + Alloc uint64 `json:"alloc"` + TotalAlloc uint64 `json:"totalAlloc"` + Sys uint64 `json:"sys"` + NumGC uint32 `json:"numGC"` + } `json:"mem"` + FS struct { + Music *FSInfo `json:"music,omitempty"` + Data *FSInfo `json:"data,omitempty"` + Cache *FSInfo `json:"cache,omitempty"` + Backup *FSInfo `json:"backup,omitempty"` + } `json:"fs"` + Library struct { + Tracks int64 `json:"tracks"` + Albums int64 `json:"albums"` + Artists int64 `json:"artists"` + Playlists int64 `json:"playlists"` + Shares int64 `json:"shares"` + Radios int64 `json:"radios"` + ActiveUsers int64 `json:"activeUsers"` + } `json:"library"` + Config struct { + LogLevel string `json:"logLevel,omitempty"` + LogFileConfigured bool `json:"logFileConfigured,omitempty"` + TLSConfigured bool `json:"tlsConfigured,omitempty"` + ScanSchedule string `json:"scanSchedule,omitempty"` + TranscodingCacheSize string `json:"transcodingCacheSize,omitempty"` + ImageCacheSize string `json:"imageCacheSize,omitempty"` + EnableArtworkPrecache bool `json:"enableArtworkPrecache,omitempty"` + EnableDownloads bool `json:"enableDownloads,omitempty"` + EnableSharing bool `json:"enableSharing,omitempty"` + EnableStarRating bool `json:"enableStarRating,omitempty"` + EnableLastFM bool `json:"enableLastFM,omitempty"` + EnableListenBrainz bool `json:"enableListenBrainz,omitempty"` + EnableMediaFileCoverArt bool `json:"enableMediaFileCoverArt,omitempty"` + EnableSpotify bool `json:"enableSpotify,omitempty"` + EnableJukebox bool `json:"enableJukebox,omitempty"` + EnablePrometheus bool `json:"enablePrometheus,omitempty"` + EnableCoverAnimation bool `json:"enableCoverAnimation,omitempty"` + SessionTimeout uint64 `json:"sessionTimeout,omitempty"` + SearchFullString bool `json:"searchFullString,omitempty"` + RecentlyAddedByModTime bool `json:"recentlyAddedByModTime,omitempty"` + PreferSortTags bool `json:"preferSortTags,omitempty"` + BackupSchedule string `json:"backupSchedule,omitempty"` + BackupCount int `json:"backupCount,omitempty"` + DevActivityPanel bool `json:"devActivityPanel,omitempty"` + DefaultBackgroundURLSet bool `json:"defaultBackgroundURL,omitempty"` + } `json:"config"` +} + +type FSInfo struct { + Type string `json:"type,omitempty"` +} diff --git a/core/metrics/insights_darwin.go b/core/metrics/insights_darwin.go new file mode 100644 index 000000000..ad59182ef --- /dev/null +++ b/core/metrics/insights_darwin.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "os/exec" + "strings" + "syscall" +) + +func getOSVersion() (string, string) { + cmd := exec.Command("sw_vers", "-productVersion") + + output, err := cmd.Output() + if err != nil { + return "", "" + } + + return strings.TrimSpace(string(output)), "" +} + +func getFilesystemType(path string) (string, error) { + var stat syscall.Statfs_t + err := syscall.Statfs(path, &stat) + if err != nil { + return "", err + } + + // Convert the filesystem type name from [16]int8 to string + fsType := make([]byte, 0, 16) + for _, c := range stat.Fstypename { + if c == 0 { + break + } + fsType = append(fsType, byte(c)) + } + + return string(fsType), nil +} diff --git a/core/metrics/insights_default.go b/core/metrics/insights_default.go new file mode 100644 index 000000000..98c34565b --- /dev/null +++ b/core/metrics/insights_default.go @@ -0,0 +1,9 @@ +//go:build !linux && !windows && !darwin + +package metrics + +import "errors" + +func getOSVersion() (string, string) { return "", "" } + +func getFilesystemType(_ string) (string, error) { return "", errors.New("not implemented") } diff --git a/core/metrics/insights_linux.go b/core/metrics/insights_linux.go new file mode 100644 index 000000000..1e732f992 --- /dev/null +++ b/core/metrics/insights_linux.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "fmt" + "io" + "os" + "strings" + "syscall" +) + +func getOSVersion() (string, string) { + file, err := os.Open("/etc/os-release") + if err != nil { + return "", "" + } + defer file.Close() + + osRelease, err := io.ReadAll(file) + if err != nil { + return "", "" + } + + lines := strings.Split(string(osRelease), "\n") + version := "" + distro := "" + for _, line := range lines { + if strings.HasPrefix(line, "VERSION_ID=") { + version = strings.ReplaceAll(strings.TrimPrefix(line, "VERSION_ID="), "\"", "") + } + if strings.HasPrefix(line, "ID=") { + distro = strings.ReplaceAll(strings.TrimPrefix(line, "ID="), "\"", "") + } + } + return version, distro +} + +// MountInfo represents an entry from /proc/self/mountinfo +type MountInfo struct { + MountPoint string + FSType string +} + +var fsTypeMap = map[int64]string{ + // Add filesystem type mappings + 0x9123683E: "btrfs", + 0x0000EF53: "ext2/ext3/ext4", + 0x00006969: "nfs", + 0x58465342: "xfs", + 0x2FC12FC1: "zfs", + 0x01021994: "tmpfs", + 0x28cd3d45: "cramfs", + 0x64626720: "debugfs", + 0x73717368: "squashfs", + 0x62656572: "sysfs", + 0x9fa0: "proc", + 0x61756673: "aufs", + 0x794c7630: "overlayfs", + 0x6a656a63: "fakeowner", // FS inside a container + // Include other filesystem types as needed +} + +func getFilesystemType(path string) (string, error) { + var fsStat syscall.Statfs_t + err := syscall.Statfs(path, &fsStat) + if err != nil { + return "", err + } + + fsType := fsStat.Type + + fsName, exists := fsTypeMap[int64(fsType)] //nolint:unconvert + if !exists { + fsName = fmt.Sprintf("unknown(0x%x)", fsType) + } + + return fsName, nil +} diff --git a/core/metrics/insights_windows.go b/core/metrics/insights_windows.go new file mode 100644 index 000000000..a2584ce54 --- /dev/null +++ b/core/metrics/insights_windows.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "os/exec" + "regexp" + + "golang.org/x/sys/windows" +) + +// Ex: Microsoft Windows [Version 10.0.26100.1742] +var winVerRegex = regexp.MustCompile(`Microsoft Windows \[Version ([\d\.]+)\]`) + +func getOSVersion() (version string, _ string) { + cmd := exec.Command("cmd", "/c", "ver") + + output, err := cmd.Output() + if err != nil { + return "", "" + } + + matches := winVerRegex.FindStringSubmatch(string(output)) + if len(matches) != 2 { + return string(output), "" + } + return matches[1], "" +} + +func getFilesystemType(path string) (string, error) { + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", err + } + + var volumeName, filesystemName [windows.MAX_PATH + 1]uint16 + var serialNumber uint32 + var maxComponentLen, filesystemFlags uint32 + + err = windows.GetVolumeInformation( + pathPtr, + &volumeName[0], + windows.MAX_PATH, + &serialNumber, + &maxComponentLen, + &filesystemFlags, + &filesystemName[0], + windows.MAX_PATH) + + if err != nil { + return "", err + } + + return windows.UTF16ToString(filesystemName[:]), nil +} diff --git a/core/metrics.go b/core/metrics/prometheus.go similarity index 99% rename from core/metrics.go rename to core/metrics/prometheus.go index bcbd08337..0f307ad76 100644 --- a/core/metrics.go +++ b/core/metrics/prometheus.go @@ -1,4 +1,4 @@ -package core +package metrics import ( "context" diff --git a/core/wire_providers.go b/core/wire_providers.go index f4231814a..2a1a71dbe 100644 --- a/core/wire_providers.go +++ b/core/wire_providers.go @@ -4,6 +4,7 @@ import ( "github.com/google/wire" "github.com/navidrome/navidrome/core/agents" "github.com/navidrome/navidrome/core/ffmpeg" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/core/playback" "github.com/navidrome/navidrome/core/scrobbler" ) @@ -20,4 +21,5 @@ var Set = wire.NewSet( ffmpeg.New, scrobbler.GetPlayTracker, playback.GetInstance, + metrics.GetInstance, ) diff --git a/go.mod b/go.mod index 2f20793f0..1c71441b1 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( golang.org/x/image v0.23.0 golang.org/x/net v0.32.0 golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 golang.org/x/text v0.21.0 golang.org/x/time v0.8.0 gopkg.in/yaml.v3 v3.0.1 @@ -105,7 +106,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.30.0 // indirect - golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.27.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/model/share.go b/model/share.go index e63df3b12..4b73b9f91 100644 --- a/model/share.go +++ b/model/share.go @@ -52,4 +52,5 @@ type ShareRepository interface { Exists(id string) (bool, error) Get(id string) (*Share, error) GetAll(options ...QueryOptions) (Shares, error) + CountAll(options ...QueryOptions) (int64, error) } diff --git a/persistence/share_repository.go b/persistence/share_repository.go index 692e61403..9177f2f06 100644 --- a/persistence/share_repository.go +++ b/persistence/share_repository.go @@ -46,6 +46,7 @@ func (r *shareRepository) selectShare(options ...model.QueryOptions) SelectBuild func (r *shareRepository) Exists(id string) (bool, error) { return r.exists(Select().Where(Eq{"id": id})) } + func (r *shareRepository) Get(id string) (*model.Share, error) { sel := r.selectShare().Where(Eq{"share.id": id}) var res model.Share diff --git a/resources/i18n/pt.json b/resources/i18n/pt.json index 1d93d3cb7..6156eff96 100644 --- a/resources/i18n/pt.json +++ b/resources/i18n/pt.json @@ -212,7 +212,8 @@ "password": "Senha", "sign_in": "Entrar", "sign_in_error": "Erro na autenticação, tente novamente.", - "logout": "Sair" + "logout": "Sair", + "insightsCollectionNote": "Navidrome coleta dados de uso anônimos para\najudar a melhorar o projeto. Clique [aqui] para\nsaber mais e para desativar se desejar" }, "validation": { "invalidChars": "Somente use letras e numeros", @@ -433,7 +434,8 @@ "links": { "homepage": "Website", "source": "Código fonte", - "featureRequests": "Solicitar funcionalidade" + "featureRequests": "Solicitar funcionalidade", + "lastInsightsCollection": "Última coleta de dados" } }, "activity": { diff --git a/scanner/scanner.go b/scanner/scanner.go index 1e7dee417..3669c88fa 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -10,6 +10,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/core/artwork" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/server/events" @@ -209,10 +210,10 @@ func (s *scanner) RescanAll(ctx context.Context, fullRescan bool) error { } if hasError { log.Error(ctx, "Errors while scanning media. Please check the logs") - core.WriteAfterScanMetrics(ctx, s.ds, false) + metrics.WriteAfterScanMetrics(ctx, s.ds, false) return ErrScanError } - core.WriteAfterScanMetrics(ctx, s.ds, true) + metrics.WriteAfterScanMetrics(ctx, s.ds, true) return nil } diff --git a/server/nativeapi/native_api.go b/server/nativeapi/native_api.go index ab0f2874f..2475862d3 100644 --- a/server/nativeapi/native_api.go +++ b/server/nativeapi/native_api.go @@ -3,11 +3,13 @@ package nativeapi import ( "context" "net/http" + "strconv" "github.com/deluan/rest" "github.com/go-chi/chi/v5" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/server" ) @@ -17,10 +19,11 @@ type Router struct { ds model.DataStore share core.Share playlists core.Playlists + insights metrics.Insights } -func New(ds model.DataStore, share core.Share, playlists core.Playlists) *Router { - r := &Router{ds: ds, share: share, playlists: playlists} +func New(ds model.DataStore, share core.Share, playlists core.Playlists, insights metrics.Insights) *Router { + r := &Router{ds: ds, share: share, playlists: playlists, insights: insights} r.Handler = r.routes() return r } @@ -55,6 +58,16 @@ func (n *Router) routes() http.Handler { r.Get("/keepalive/*", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"response":"ok", "id":"keepalive"}`)) }) + + // Insights status endpoint + r.Get("/insights/*", func(w http.ResponseWriter, r *http.Request) { + last, success := n.insights.LastRun(r.Context()) + if conf.Server.EnableInsightsCollector { + _, _ = w.Write([]byte(`{"id":"insights_status", "lastRun":"` + last.Format("2006-01-02 15:04:05") + `", "success":` + strconv.FormatBool(success) + `}`)) + } else { + _, _ = w.Write([]byte(`{"id":"insights_status", "lastRun":"disabled", "success":false}`)) + } + }) }) return r diff --git a/server/server.go b/server/server.go index a8a89db5b..2c2129afc 100644 --- a/server/server.go +++ b/server/server.go @@ -20,6 +20,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/core/auth" + "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/server/events" @@ -27,14 +28,15 @@ import ( ) type Server struct { - router chi.Router - ds model.DataStore - appRoot string - broker events.Broker + router chi.Router + ds model.DataStore + appRoot string + broker events.Broker + insights metrics.Insights } -func New(ds model.DataStore, broker events.Broker) *Server { - s := &Server{ds: ds, broker: broker} +func New(ds model.DataStore, broker events.Broker, insights metrics.Insights) *Server { + s := &Server{ds: ds, broker: broker, insights: insights} initialSetup(ds) auth.Init(s.ds) s.initRoutes() diff --git a/ui/src/App.jsx b/ui/src/App.jsx index 9ff15f377..41cfb6186 100644 --- a/ui/src/App.jsx +++ b/ui/src/App.jsx @@ -123,6 +123,7 @@ const Admin = (props) => { , , , + , , ]} diff --git a/ui/src/consts.js b/ui/src/consts.js index b0524669e..e3446c2fe 100644 --- a/ui/src/consts.js +++ b/ui/src/consts.js @@ -1,5 +1,8 @@ export const REST_URL = '/api' +export const INSIGHTS_DOC_URL = + 'https://navidrome.org/docs/getting-started/insights' + export const M3U_MIME_TYPE = 'audio/x-mpegurl' export const AUTO_THEME_ID = 'AUTO_THEME_ID' diff --git a/ui/src/dialogs/AboutDialog.jsx b/ui/src/dialogs/AboutDialog.jsx index 046e2eb77..baee011ca 100644 --- a/ui/src/dialogs/AboutDialog.jsx +++ b/ui/src/dialogs/AboutDialog.jsx @@ -11,10 +11,11 @@ import TableCell from '@material-ui/core/TableCell' import Paper from '@material-ui/core/Paper' import FavoriteBorderIcon from '@material-ui/icons/FavoriteBorder' import inflection from 'inflection' -import { useTranslate } from 'react-admin' +import { useGetOne, usePermissions, useTranslate } from 'react-admin' import config from '../config' import { DialogTitle } from './DialogTitle' import { DialogContent } from './DialogContent' +import { INSIGHTS_DOC_URL } from '../consts.js' const links = { homepage: 'navidrome.org', @@ -51,6 +52,9 @@ const LinkToVersion = ({ version }) => { const AboutDialog = ({ open, onClose }) => { const translate = useTranslate() + const { permissions } = usePermissions() + const { data, loading } = useGetOne('insights', 'insights_status') + return ( @@ -87,6 +91,18 @@ const AboutDialog = ({ open, onClose }) => { ) })} + {permissions === 'admin' ? ( + + + {translate(`about.links.lastInsightsCollection`)}: + + + + {(!loading && data?.lastRun) || 'N/A'}{' '} + + + + ) : null} ({ @@ -81,6 +83,13 @@ const useStyles = makeStyles( systemNameLink: { textDecoration: 'none', }, + message: { + marginTop: '1em', + padding: '0 1em 1em 1em', + textAlign: 'center', + wordBreak: 'break-word', + fontSize: '0.875em', + }, }), { name: 'NDLogin' }, ) @@ -173,6 +182,62 @@ const FormLogin = ({ loading, handleSubmit, validate }) => { ) } +const InsightsNotice = ({ url }) => { + const translate = useTranslate() + const classes = useStyles() + + const anchorRegex = /\[(.+?)]/g + const originalMsg = translate('ra.auth.insightsCollectionNote') + + // Split the entire message on newlines + const lines = originalMsg.split('\n') + + const renderedLines = lines.map((line, lineIndex) => { + const segments = [] + let lastIndex = 0 + let match + + // Find bracketed text in each line + while ((match = anchorRegex.exec(line)) !== null) { + // match.index is where "[something]" starts + // match[1] is the text inside the brackets + const bracketText = match[1] + + // Push the text before the bracket + segments.push(line.slice(lastIndex, match.index)) + + // Push the component + segments.push( + + {bracketText} + , + ) + + // Update lastIndex to the character right after the bracketed text + lastIndex = match.index + match[0].length + } + + // Push the remaining text after the last bracket + segments.push(line.slice(lastIndex)) + + // Return this line’s parts, plus a
if not the last line + return ( + + {segments} + {lineIndex < lines.length - 1 &&
} + + ) + }) + + return
{renderedLines}
+} + const FormSignUp = ({ loading, handleSubmit, validate }) => { const translate = useTranslate() const classes = useStyles() @@ -237,6 +302,7 @@ const FormSignUp = ({ loading, handleSubmit, validate }) => { {translate('ra.auth.buttonCreateAdmin')} + @@ -245,6 +311,7 @@ const FormSignUp = ({ loading, handleSubmit, validate }) => { /> ) } + const Login = ({ location }) => { const [loading, setLoading] = useState(false) const translate = useTranslate()