From fc3a8134ca6d770a47793e375f6e77803d3b0aa7 Mon Sep 17 00:00:00 2001 From: Josh Patra Date: Mon, 2 Feb 2026 12:24:30 -0500 Subject: [PATCH] spotify track prematching --- .env.example | 4 + SPOTIFY_MATCHING_OPTIMIZATION.md | 0 allstarr/Controllers/JellyfinController.cs | 59 +++--- allstarr/Program.cs | 3 + .../Spotify/SpotifyTrackMatchingService.cs | 189 ++++++++++++++++++ docker-compose.yml | 4 +- 6 files changed, 229 insertions(+), 30 deletions(-) create mode 100644 SPOTIFY_MATCHING_OPTIMIZATION.md create mode 100644 allstarr/Services/Spotify/SpotifyTrackMatchingService.cs diff --git a/.env.example b/.env.example index 5824c1d..3ea1037 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ BACKEND_TYPE=Subsonic # Enable Redis caching for metadata and images (default: true) REDIS_ENABLED=true +# Redis data persistence directory (default: ./redis-data) +# Redis will save snapshots and append-only logs here to persist cache across restarts +REDIS_DATA_PATH=./redis-data + # ===== SUBSONIC/NAVIDROME CONFIGURATION ===== # Server URL (required if using Subsonic backend) SUBSONIC_URL=http://localhost:4533 diff --git a/SPOTIFY_MATCHING_OPTIMIZATION.md b/SPOTIFY_MATCHING_OPTIMIZATION.md new file mode 100644 index 0000000..e69de29 diff --git a/allstarr/Controllers/JellyfinController.cs b/allstarr/Controllers/JellyfinController.cs index cd161c3..c6e8dbd 100644 --- a/allstarr/Controllers/JellyfinController.cs +++ b/allstarr/Controllers/JellyfinController.cs @@ -2804,20 +2804,22 @@ public class JellyfinController : ControllerBase _logger.LogInformation("Matching {Count} missing tracks for {Playlist}", missingTracks.Count, spotifyPlaylistName); - // Match missing tracks (excluding ones we already have locally) - var matchTasks = missingTracks + // Match missing tracks sequentially with rate limiting (excluding ones we already have locally) + var matchedBySpotifyId = new Dictionary(); + var tracksToMatch = missingTracks .Where(track => !existingSpotifyIds.Contains(track.SpotifyId)) - .Select(async track => + .ToList(); + + foreach (var track in tracksToMatch) + { + try { - try + // Search with just title and artist for better matching + var query = $"{track.Title} {track.PrimaryArtist}"; + var results = await _metadataService.SearchSongsAsync(query, limit: 5); + + if (results.Count > 0) { - // Search with just title and artist for better matching - var query = $"{track.Title} {track.PrimaryArtist}"; - var results = await _metadataService.SearchSongsAsync(query, limit: 5); - - if (results.Count == 0) - return (track.SpotifyId, (Song?)null); - // Fuzzy match to find best result var bestMatch = results .Select(song => new @@ -2837,32 +2839,31 @@ public class JellyfinController : ControllerBase .OrderByDescending(x => x.TotalScore) .FirstOrDefault(); - // Only return if match is good enough (>60% combined score) + // Only add if match is good enough (>60% combined score) if (bestMatch != null && bestMatch.TotalScore >= 60) { _logger.LogDebug("Matched '{Title}' by {Artist} -> '{MatchTitle}' by {MatchArtist} (score: {Score:F1})", track.Title, track.PrimaryArtist, bestMatch.Song.Title, bestMatch.Song.Artist, bestMatch.TotalScore); - return (track.SpotifyId, (Song?)bestMatch.Song); + matchedBySpotifyId[track.SpotifyId] = bestMatch.Song; + } + else + { + _logger.LogDebug("No good match for '{Title}' by {Artist} (best score: {Score:F1})", + track.Title, track.PrimaryArtist, bestMatch?.TotalScore ?? 0); } - - _logger.LogDebug("No good match for '{Title}' by {Artist} (best score: {Score:F1})", - track.Title, track.PrimaryArtist, bestMatch?.TotalScore ?? 0); - return (track.SpotifyId, (Song?)null); } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", - track.Title, track.PrimaryArtist); - return (track.SpotifyId, (Song?)null); - } - }); - - var matchResults = await Task.WhenAll(matchTasks); - var matchedBySpotifyId = matchResults - .Where(x => x.Item2 != null) - .ToDictionary(x => x.SpotifyId, x => x.Item2!); + + // Rate limiting: small delay between searches to avoid overwhelming the service + await Task.Delay(100); // 100ms delay = max 10 searches/second + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", + track.Title, track.PrimaryArtist); + } + } // Build final track list in Spotify playlist order var finalTracks = new List(); diff --git a/allstarr/Program.cs b/allstarr/Program.cs index 7586869..1e45c12 100644 --- a/allstarr/Program.cs +++ b/allstarr/Program.cs @@ -269,6 +269,9 @@ builder.Services.AddHostedService(); // Register Spotify missing tracks fetcher (only runs when SpotifyImport is enabled) builder.Services.AddHostedService(); +// Register Spotify track matching service (pre-matches tracks with rate limiting) +builder.Services.AddHostedService(); + builder.Services.AddCors(options => { options.AddDefaultPolicy(policy => diff --git a/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs b/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs new file mode 100644 index 0000000..f00f2c5 --- /dev/null +++ b/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs @@ -0,0 +1,189 @@ +using allstarr.Models.Domain; +using allstarr.Models.Settings; +using allstarr.Models.Spotify; +using allstarr.Services.Common; +using Microsoft.Extensions.Options; +using System.Text.Json; + +namespace allstarr.Services.Spotify; + +/// +/// Background service that pre-matches Spotify missing tracks with external providers. +/// Runs after SpotifyMissingTracksFetcher completes to avoid rate limiting during playlist loading. +/// +public class SpotifyTrackMatchingService : BackgroundService +{ + private readonly IOptions _spotifySettings; + private readonly RedisCacheService _cache; + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + private const int DelayBetweenSearchesMs = 150; // 150ms = ~6.6 searches/second to avoid rate limiting + + public SpotifyTrackMatchingService( + IOptions spotifySettings, + RedisCacheService cache, + IServiceProvider serviceProvider, + ILogger logger) + { + _spotifySettings = spotifySettings; + _cache = cache; + _serviceProvider = serviceProvider; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("SpotifyTrackMatchingService: Starting up..."); + + if (!_spotifySettings.Value.Enabled) + { + _logger.LogInformation("Spotify playlist injection is DISABLED, matching service will not run"); + return; + } + + // Wait a bit for the fetcher to run first + await Task.Delay(TimeSpan.FromMinutes(2), stoppingToken); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await MatchAllPlaylistsAsync(stoppingToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error in track matching service"); + } + + // Run every 30 minutes to catch new missing tracks + await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken); + } + } + + private async Task MatchAllPlaylistsAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("=== STARTING TRACK MATCHING ==="); + + var playlistNames = _spotifySettings.Value.PlaylistNames; + if (playlistNames.Count == 0) + { + _logger.LogInformation("No playlists configured for matching"); + return; + } + + using var scope = _serviceProvider.CreateScope(); + var metadataService = scope.ServiceProvider.GetRequiredService(); + + foreach (var playlistName in playlistNames) + { + if (cancellationToken.IsCancellationRequested) break; + + try + { + await MatchPlaylistTracksAsync(playlistName, metadataService, cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error matching tracks for playlist {Playlist}", playlistName); + } + } + + _logger.LogInformation("=== FINISHED TRACK MATCHING ==="); + } + + private async Task MatchPlaylistTracksAsync( + string playlistName, + IMusicMetadataService metadataService, + CancellationToken cancellationToken) + { + var missingTracksKey = $"spotify:missing:{playlistName}"; + var matchedTracksKey = $"spotify:matched:{playlistName}"; + + // Check if we already have matched tracks cached + var existingMatched = await _cache.GetAsync>(matchedTracksKey); + if (existingMatched != null && existingMatched.Count > 0) + { + _logger.LogDebug("Playlist {Playlist} already has {Count} matched tracks cached, skipping", + playlistName, existingMatched.Count); + return; + } + + // Get missing tracks + var missingTracks = await _cache.GetAsync>(missingTracksKey); + if (missingTracks == null || missingTracks.Count == 0) + { + _logger.LogDebug("No missing tracks found for {Playlist}, skipping matching", playlistName); + return; + } + + _logger.LogInformation("Matching {Count} tracks for {Playlist} (with rate limiting)", + missingTracks.Count, playlistName); + + var matchedSongs = new List(); + var matchCount = 0; + + foreach (var track in missingTracks) + { + if (cancellationToken.IsCancellationRequested) break; + + try + { + var query = $"{track.Title} {track.PrimaryArtist}"; + var results = await metadataService.SearchSongsAsync(query, limit: 5); + + if (results.Count > 0) + { + // Fuzzy match to find best result + var bestMatch = results + .Select(song => new + { + Song = song, + TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title), + ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist) + }) + .Select(x => new + { + x.Song, + x.TitleScore, + x.ArtistScore, + TotalScore = (x.TitleScore * 0.6) + (x.ArtistScore * 0.4) + }) + .OrderByDescending(x => x.TotalScore) + .FirstOrDefault(); + + if (bestMatch != null && bestMatch.TotalScore >= 60) + { + matchedSongs.Add(bestMatch.Song); + matchCount++; + + if (matchCount % 10 == 0) + { + _logger.LogDebug("Matched {Count}/{Total} tracks for {Playlist}", + matchCount, missingTracks.Count, playlistName); + } + } + } + + // Rate limiting: delay between searches + await Task.Delay(DelayBetweenSearchesMs, cancellationToken); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", + track.Title, track.PrimaryArtist); + } + } + + if (matchedSongs.Count > 0) + { + // Cache matched tracks for 1 hour + await _cache.SetAsync(matchedTracksKey, matchedSongs, TimeSpan.FromHours(1)); + _logger.LogInformation("✓ Cached {Matched}/{Total} matched tracks for {Playlist}", + matchedSongs.Count, missingTracks.Count, playlistName); + } + else + { + _logger.LogInformation("No tracks matched for {Playlist}", playlistName); + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index ec79566..fc38039 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,12 +6,14 @@ services: # Redis is only accessible internally - no external port exposure expose: - "6379" - command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru + command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru --save 60 1 --appendonly yes healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s timeout: 3s retries: 3 + volumes: + - ${REDIS_DATA_PATH:-./redis-data}:/data networks: - allstarr-network