spotify track prematching

.env.example

@@ -6,6 +6,10 @@ BACKEND_TYPE=Subsonic
 # Enable Redis caching for metadata and images (default: true)
 REDIS_ENABLED=true
 
+# Redis data persistence directory (default: ./redis-data)
+# Redis will save snapshots and append-only logs here to persist cache across restarts
+REDIS_DATA_PATH=./redis-data
+
 # ===== SUBSONIC/NAVIDROME CONFIGURATION =====
 # Server URL (required if using Subsonic backend)
 SUBSONIC_URL=http://localhost:4533

allstarr/Controllers/JellyfinController.cs

@@ -2804,20 +2804,22 @@ public class JellyfinController : ControllerBase
             _logger.LogInformation("Matching {Count} missing tracks for {Playlist}", 
                 missingTracks.Count, spotifyPlaylistName);
 
-            // Match missing tracks (excluding ones we already have locally)
-            var matchTasks = missingTracks
+            // Match missing tracks sequentially with rate limiting (excluding ones we already have locally)
+            var matchedBySpotifyId = new Dictionary<string, Song>();
+            var tracksToMatch = missingTracks
                 .Where(track => !existingSpotifyIds.Contains(track.SpotifyId))
-                .Select(async track =>
+                .ToList();
+
+            foreach (var track in tracksToMatch)
+            {
+                try
                 {
-                    try
+                    // Search with just title and artist for better matching
+                    var query = $"{track.Title} {track.PrimaryArtist}";
+                    var results = await _metadataService.SearchSongsAsync(query, limit: 5);
+                    
+                    if (results.Count > 0)
                     {
-                        // Search with just title and artist for better matching
-                        var query = $"{track.Title} {track.PrimaryArtist}";
-                        var results = await _metadataService.SearchSongsAsync(query, limit: 5);
-                        
-                        if (results.Count == 0)
-                            return (track.SpotifyId, (Song?)null);
-                        
                         // Fuzzy match to find best result
                         var bestMatch = results
                             .Select(song => new
@@ -2837,32 +2839,31 @@ public class JellyfinController : ControllerBase
                             .OrderByDescending(x => x.TotalScore)
                             .FirstOrDefault();
                         
-                        // Only return if match is good enough (>60% combined score)
+                        // Only add if match is good enough (>60% combined score)
                         if (bestMatch != null && bestMatch.TotalScore >= 60)
                         {
                             _logger.LogDebug("Matched '{Title}' by {Artist} -> '{MatchTitle}' by {MatchArtist} (score: {Score:F1})",
                                 track.Title, track.PrimaryArtist, 
                                 bestMatch.Song.Title, bestMatch.Song.Artist, 
                                 bestMatch.TotalScore);
-                            return (track.SpotifyId, (Song?)bestMatch.Song);
+                            matchedBySpotifyId[track.SpotifyId] = bestMatch.Song;
+                        }
+                        else
+                        {
+                            _logger.LogDebug("No good match for '{Title}' by {Artist} (best score: {Score:F1})",
+                                track.Title, track.PrimaryArtist, bestMatch?.TotalScore ?? 0);
                         }
-                        
-                        _logger.LogDebug("No good match for '{Title}' by {Artist} (best score: {Score:F1})",
-                            track.Title, track.PrimaryArtist, bestMatch?.TotalScore ?? 0);
-                        return (track.SpotifyId, (Song?)null);
                     }
-                    catch (Exception ex)
-                    {
-                        _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", 
-                            track.Title, track.PrimaryArtist);
-                        return (track.SpotifyId, (Song?)null);
-                    }
-                });
-
-            var matchResults = await Task.WhenAll(matchTasks);
-            var matchedBySpotifyId = matchResults
-                .Where(x => x.Item2 != null)
-                .ToDictionary(x => x.SpotifyId, x => x.Item2!);
+                    
+                    // Rate limiting: small delay between searches to avoid overwhelming the service
+                    await Task.Delay(100); // 100ms delay = max 10 searches/second
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", 
+                        track.Title, track.PrimaryArtist);
+                }
+            }
 
             // Build final track list in Spotify playlist order
             var finalTracks = new List<Song>();

allstarr/Program.cs

@@ -269,6 +269,9 @@ builder.Services.AddHostedService<CacheCleanupService>();
 // Register Spotify missing tracks fetcher (only runs when SpotifyImport is enabled)
 builder.Services.AddHostedService<allstarr.Services.Spotify.SpotifyMissingTracksFetcher>();
 
+// Register Spotify track matching service (pre-matches tracks with rate limiting)
+builder.Services.AddHostedService<allstarr.Services.Spotify.SpotifyTrackMatchingService>();
+
 builder.Services.AddCors(options =>
 {
     options.AddDefaultPolicy(policy =>

allstarr/Services/Spotify/SpotifyTrackMatchingService.cs

@@ -0,0 +1,189 @@
+using allstarr.Models.Domain;
+using allstarr.Models.Settings;
+using allstarr.Models.Spotify;
+using allstarr.Services.Common;
+using Microsoft.Extensions.Options;
+using System.Text.Json;
+
+namespace allstarr.Services.Spotify;
+
+/// <summary>
+/// Background service that pre-matches Spotify missing tracks with external providers.
+/// Runs after SpotifyMissingTracksFetcher completes to avoid rate limiting during playlist loading.
+/// </summary>
+public class SpotifyTrackMatchingService : BackgroundService
+{
+    private readonly IOptions<SpotifyImportSettings> _spotifySettings;
+    private readonly RedisCacheService _cache;
+    private readonly ILogger<SpotifyTrackMatchingService> _logger;
+    private readonly IServiceProvider _serviceProvider;
+    private const int DelayBetweenSearchesMs = 150; // 150ms = ~6.6 searches/second to avoid rate limiting
+
+    public SpotifyTrackMatchingService(
+        IOptions<SpotifyImportSettings> spotifySettings,
+        RedisCacheService cache,
+        IServiceProvider serviceProvider,
+        ILogger<SpotifyTrackMatchingService> logger)
+    {
+        _spotifySettings = spotifySettings;
+        _cache = cache;
+        _serviceProvider = serviceProvider;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("SpotifyTrackMatchingService: Starting up...");
+        
+        if (!_spotifySettings.Value.Enabled)
+        {
+            _logger.LogInformation("Spotify playlist injection is DISABLED, matching service will not run");
+            return;
+        }
+
+        // Wait a bit for the fetcher to run first
+        await Task.Delay(TimeSpan.FromMinutes(2), stoppingToken);
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await MatchAllPlaylistsAsync(stoppingToken);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error in track matching service");
+            }
+
+            // Run every 30 minutes to catch new missing tracks
+            await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken);
+        }
+    }
+
+    private async Task MatchAllPlaylistsAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("=== STARTING TRACK MATCHING ===");
+        
+        var playlistNames = _spotifySettings.Value.PlaylistNames;
+        if (playlistNames.Count == 0)
+        {
+            _logger.LogInformation("No playlists configured for matching");
+            return;
+        }
+
+        using var scope = _serviceProvider.CreateScope();
+        var metadataService = scope.ServiceProvider.GetRequiredService<IMusicMetadataService>();
+
+        foreach (var playlistName in playlistNames)
+        {
+            if (cancellationToken.IsCancellationRequested) break;
+            
+            try
+            {
+                await MatchPlaylistTracksAsync(playlistName, metadataService, cancellationToken);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error matching tracks for playlist {Playlist}", playlistName);
+            }
+        }
+        
+        _logger.LogInformation("=== FINISHED TRACK MATCHING ===");
+    }
+
+    private async Task MatchPlaylistTracksAsync(
+        string playlistName,
+        IMusicMetadataService metadataService,
+        CancellationToken cancellationToken)
+    {
+        var missingTracksKey = $"spotify:missing:{playlistName}";
+        var matchedTracksKey = $"spotify:matched:{playlistName}";
+        
+        // Check if we already have matched tracks cached
+        var existingMatched = await _cache.GetAsync<List<Song>>(matchedTracksKey);
+        if (existingMatched != null && existingMatched.Count > 0)
+        {
+            _logger.LogDebug("Playlist {Playlist} already has {Count} matched tracks cached, skipping", 
+                playlistName, existingMatched.Count);
+            return;
+        }
+
+        // Get missing tracks
+        var missingTracks = await _cache.GetAsync<List<MissingTrack>>(missingTracksKey);
+        if (missingTracks == null || missingTracks.Count == 0)
+        {
+            _logger.LogDebug("No missing tracks found for {Playlist}, skipping matching", playlistName);
+            return;
+        }
+
+        _logger.LogInformation("Matching {Count} tracks for {Playlist} (with rate limiting)", 
+            missingTracks.Count, playlistName);
+
+        var matchedSongs = new List<Song>();
+        var matchCount = 0;
+
+        foreach (var track in missingTracks)
+        {
+            if (cancellationToken.IsCancellationRequested) break;
+
+            try
+            {
+                var query = $"{track.Title} {track.PrimaryArtist}";
+                var results = await metadataService.SearchSongsAsync(query, limit: 5);
+                
+                if (results.Count > 0)
+                {
+                    // Fuzzy match to find best result
+                    var bestMatch = results
+                        .Select(song => new
+                        {
+                            Song = song,
+                            TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
+                            ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist)
+                        })
+                        .Select(x => new
+                        {
+                            x.Song,
+                            x.TitleScore,
+                            x.ArtistScore,
+                            TotalScore = (x.TitleScore * 0.6) + (x.ArtistScore * 0.4)
+                        })
+                        .OrderByDescending(x => x.TotalScore)
+                        .FirstOrDefault();
+                    
+                    if (bestMatch != null && bestMatch.TotalScore >= 60)
+                    {
+                        matchedSongs.Add(bestMatch.Song);
+                        matchCount++;
+                        
+                        if (matchCount % 10 == 0)
+                        {
+                            _logger.LogDebug("Matched {Count}/{Total} tracks for {Playlist}", 
+                                matchCount, missingTracks.Count, playlistName);
+                        }
+                    }
+                }
+                
+                // Rate limiting: delay between searches
+                await Task.Delay(DelayBetweenSearchesMs, cancellationToken);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}", 
+                    track.Title, track.PrimaryArtist);
+            }
+        }
+
+        if (matchedSongs.Count > 0)
+        {
+            // Cache matched tracks for 1 hour
+            await _cache.SetAsync(matchedTracksKey, matchedSongs, TimeSpan.FromHours(1));
+            _logger.LogInformation("✓ Cached {Matched}/{Total} matched tracks for {Playlist}", 
+                matchedSongs.Count, missingTracks.Count, playlistName);
+        }
+        else
+        {
+            _logger.LogInformation("No tracks matched for {Playlist}", playlistName);
+        }
+    }
+}

docker-compose.yml

@@ -6,12 +6,14 @@ services:
     # Redis is only accessible internally - no external port exposure
     expose:
       - "6379"
-    command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru
+    command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru --save 60 1 --appendonly yes
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s
       timeout: 3s
       retries: 3
+    volumes:
+      - ${REDIS_DATA_PATH:-./redis-data}:/data
     networks:
       - allstarr-network