From 3e5c57766b9e9f3a86ff501dc4051dfeae061f9b Mon Sep 17 00:00:00 2001 From: Josh Patra Date: Wed, 4 Feb 2026 17:03:50 -0500 Subject: [PATCH] feat: pre-build playlist cache and make matching interval configurable - Pre-build playlist items cache during track matching for instant serving - Add PreBuildPlaylistItemsCacheAsync() to SpotifyTrackMatchingService - Combines local Jellyfin tracks + external matched tracks in correct Spotify order - Saves to both Redis and file cache for persistence across restarts - Change matching interval from hardcoded 30 minutes to configurable (default: 24 hours) - Add SPOTIFY_IMPORT_MATCHING_INTERVAL_HOURS environment variable - Set to 0 to only run once on startup (manual trigger still works) - Add endpoint usage files to .gitignore - Update documentation in README and .env.example Rationale: Spotify playlists like Discover Weekly update once per week, so running every 24 hours is more than sufficient. Pre-building the cache eliminates slow 'on the fly' playlist building. All 225 tests pass. --- .env.example | 7 + .gitignore | 4 + README.md | 5 +- .../Models/Settings/SpotifyImportSettings.cs | 9 + .../Spotify/SpotifyTrackMatchingService.cs | 202 +++++++++++++++++- allstarr/appsettings.json | 1 + 6 files changed, 225 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 2489a77..8b9b972 100644 --- a/.env.example +++ b/.env.example @@ -126,6 +126,13 @@ SPOTIFY_IMPORT_SYNC_START_MINUTE=15 # Example: If plugin runs at 4:15 PM and window is 2 hours, checks from 4:00 PM to 6:00 PM SPOTIFY_IMPORT_SYNC_WINDOW_HOURS=2 +# Matching interval: How often to run track matching (in hours) +# Spotify playlists like Discover Weekly update once per week, Release Radar updates weekly +# Most playlists don't change frequently, so running once per day is reasonable +# Set to 0 to only run once on startup (manual trigger via admin UI still works) +# Default: 24 hours +SPOTIFY_IMPORT_MATCHING_INTERVAL_HOURS=24 + # Playlists configuration (JSON ARRAY FORMAT - managed by web UI) # Format: [["PlaylistName","SpotifyPlaylistId","first|last"],...] # - PlaylistName: Name as it appears in Jellyfin diff --git a/.gitignore b/.gitignore index b52896f..1652d08 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,10 @@ apis/*.md apis/*.json !apis/jellyfin-openapi-stable.json +# Endpoint usage tracking +apis/endpoint-usage.json +/app/cache/endpoint-usage/ + # Original source code for reference originals/ diff --git a/README.md b/README.md index c6917b7..e0dad46 100644 --- a/README.md +++ b/README.md @@ -401,11 +401,14 @@ SPOTIFY_IMPORT_PLAYLIST_NAMES=Release Radar,Discover Weekly - Caches the list of missing tracks in Redis + file cache - Runs automatically on startup (if needed) and every 5 minutes during the sync window -3. **Allstarr Matches Tracks** (2 minutes after startup, then every 30 minutes) +3. **Allstarr Matches Tracks** (2 minutes after startup, then configurable interval) - For each missing track, searches your streaming provider (SquidWTF, Deezer, or Qobuz) - Uses fuzzy matching to find the best match (title + artist similarity) - Rate-limited to avoid overwhelming the service (150ms delay between searches) - Caches matched results for 1 hour + - **Pre-builds playlist items cache** for instant serving (no "on the fly" building) + - Default interval: 24 hours (configurable via `SPOTIFY_IMPORT_MATCHING_INTERVAL_HOURS`) + - Set to 0 to only run once on startup (manual trigger via admin UI still works) 4. **You Open the Playlist in Jellyfin** - Allstarr intercepts the request diff --git a/allstarr/Models/Settings/SpotifyImportSettings.cs b/allstarr/Models/Settings/SpotifyImportSettings.cs index 44d2768..4296141 100644 --- a/allstarr/Models/Settings/SpotifyImportSettings.cs +++ b/allstarr/Models/Settings/SpotifyImportSettings.cs @@ -80,6 +80,15 @@ public class SpotifyImportSettings /// public int SyncWindowHours { get; set; } = 2; + /// + /// How often to run track matching in hours. + /// Spotify playlists like Discover Weekly update once per week, Release Radar updates weekly. + /// Most playlists don't change frequently, so running every 24 hours is reasonable. + /// Set to 0 to only run once on startup (manual trigger via admin UI still works). + /// Default: 24 hours + /// + public int MatchingIntervalHours { get; set; } = 24; + /// /// Combined playlist configuration as JSON array. /// Format: [["Name","Id","first|last"],...] diff --git a/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs b/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs index 14f6a0d..d44c992 100644 --- a/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs +++ b/allstarr/Services/Spotify/SpotifyTrackMatchingService.cs @@ -72,8 +72,15 @@ public class SpotifyTrackMatchingService : BackgroundService // Now start the periodic matching loop while (!stoppingToken.IsCancellationRequested) { - // Wait 30 minutes before next run - await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken); + // Wait for configured interval before next run (default 24 hours) + var intervalHours = _spotifySettings.MatchingIntervalHours; + if (intervalHours <= 0) + { + _logger.LogInformation("Periodic matching disabled (MatchingIntervalHours = {Hours}), only startup run will execute", intervalHours); + break; // Exit loop - only run once on startup + } + + await Task.Delay(TimeSpan.FromHours(intervalHours), stoppingToken); try { @@ -421,6 +428,9 @@ public class SpotifyTrackMatchingService : BackgroundService _logger.LogInformation( "✓ Cached {Matched}/{Total} tracks for {Playlist} (ISRC: {Isrc}, Fuzzy: {Fuzzy}, No match: {NoMatch})", matchedTracks.Count, tracksToMatch.Count, playlistName, isrcMatches, fuzzyMatches, noMatch); + + // Pre-build playlist items cache for instant serving + await PreBuildPlaylistItemsCacheAsync(playlistName, playlistConfig?.JellyfinId, spotifyTracks, matchedTracks, cancellationToken); } else { @@ -650,4 +660,192 @@ public class SpotifyTrackMatchingService : BackgroundService return avgScore; } + + /// + /// Pre-builds the playlist items cache for instant serving. + /// This combines local Jellyfin tracks with external matched tracks in the correct Spotify order. + /// + private async Task PreBuildPlaylistItemsCacheAsync( + string playlistName, + string? jellyfinPlaylistId, + List spotifyTracks, + List matchedTracks, + CancellationToken cancellationToken) + { + try + { + _logger.LogInformation("🔨 Pre-building playlist items cache for {Playlist}...", playlistName); + + if (string.IsNullOrEmpty(jellyfinPlaylistId)) + { + _logger.LogWarning("No Jellyfin playlist ID configured for {Playlist}, cannot pre-build cache", playlistName); + return; + } + + // Get existing tracks from Jellyfin playlist + using var scope = _serviceProvider.CreateScope(); + var proxyService = scope.ServiceProvider.GetService(); + var responseBuilder = scope.ServiceProvider.GetService(); + var jellyfinSettings = scope.ServiceProvider.GetService>()?.Value; + + if (proxyService == null || responseBuilder == null || jellyfinSettings == null) + { + _logger.LogWarning("Required services not available for pre-building cache"); + return; + } + + var userId = jellyfinSettings.UserId; + if (string.IsNullOrEmpty(userId)) + { + _logger.LogWarning("No UserId configured, cannot pre-build playlist cache for {Playlist}", playlistName); + return; + } + + var playlistItemsUrl = $"Playlists/{jellyfinPlaylistId}/Items?UserId={userId}&Fields=MediaSources"; + var (existingTracksResponse, statusCode) = await proxyService.GetJsonAsync(playlistItemsUrl, null, null); + + if (statusCode != 200 || existingTracksResponse == null) + { + _logger.LogWarning("Failed to fetch Jellyfin playlist items for {Playlist}: HTTP {StatusCode}", playlistName, statusCode); + return; + } + + // Index Jellyfin items by title+artist for matching + var jellyfinItemsByName = new Dictionary(); + + if (existingTracksResponse.RootElement.TryGetProperty("Items", out var items)) + { + foreach (var item in items.EnumerateArray()) + { + var title = item.TryGetProperty("Name", out var nameEl) ? nameEl.GetString() ?? "" : ""; + var artist = ""; + if (item.TryGetProperty("Artists", out var artistsEl) && artistsEl.GetArrayLength() > 0) + { + artist = artistsEl[0].GetString() ?? ""; + } + else if (item.TryGetProperty("AlbumArtist", out var albumArtistEl)) + { + artist = albumArtistEl.GetString() ?? ""; + } + + var key = $"{title}|{artist}".ToLowerInvariant(); + if (!jellyfinItemsByName.ContainsKey(key)) + { + jellyfinItemsByName[key] = item; + } + } + } + + // Build the final track list in correct Spotify order + var finalItems = new List>(); + var usedJellyfinItems = new HashSet(); + var localUsedCount = 0; + var externalUsedCount = 0; + + foreach (var spotifyTrack in spotifyTracks.OrderBy(t => t.Position)) + { + if (cancellationToken.IsCancellationRequested) break; + + // Try to find matching Jellyfin item by fuzzy matching + JsonElement? matchedJellyfinItem = null; + string? matchedKey = null; + double bestScore = 0; + + foreach (var kvp in jellyfinItemsByName) + { + if (usedJellyfinItems.Contains(kvp.Key)) continue; + + var item = kvp.Value; + var title = item.TryGetProperty("Name", out var nameEl) ? nameEl.GetString() ?? "" : ""; + var artist = ""; + if (item.TryGetProperty("Artists", out var artistsEl) && artistsEl.GetArrayLength() > 0) + { + artist = artistsEl[0].GetString() ?? ""; + } + + var titleScore = FuzzyMatcher.CalculateSimilarity(spotifyTrack.Title, title); + var artistScore = FuzzyMatcher.CalculateSimilarity(spotifyTrack.PrimaryArtist, artist); + var totalScore = (titleScore * 0.7) + (artistScore * 0.3); + + if (totalScore > bestScore && totalScore >= 70) + { + bestScore = totalScore; + matchedJellyfinItem = item; + matchedKey = kvp.Key; + } + } + + if (matchedJellyfinItem.HasValue && matchedKey != null) + { + // Use the raw Jellyfin item (preserves ALL metadata) + var itemDict = JsonSerializer.Deserialize>(matchedJellyfinItem.Value.GetRawText()); + if (itemDict != null) + { + finalItems.Add(itemDict); + usedJellyfinItems.Add(matchedKey); + localUsedCount++; + } + } + else + { + // No local match - try to find external track + var matched = matchedTracks.FirstOrDefault(t => t.SpotifyId == spotifyTrack.SpotifyId); + if (matched != null && matched.MatchedSong != null) + { + // Convert external song to Jellyfin item format + var externalItem = responseBuilder.ConvertSongToJellyfinItem(matched.MatchedSong); + finalItems.Add(externalItem); + externalUsedCount++; + } + } + } + + if (finalItems.Count > 0) + { + // Save to Redis cache + var cacheKey = $"spotify:playlist:items:{playlistName}"; + await _cache.SetAsync(cacheKey, finalItems, TimeSpan.FromHours(24)); + + // Save to file cache for persistence + await SavePlaylistItemsToFileAsync(playlistName, finalItems); + + _logger.LogInformation( + "✅ Pre-built playlist cache for {Playlist}: {Total} tracks ({Local} LOCAL + {External} EXTERNAL)", + playlistName, finalItems.Count, localUsedCount, externalUsedCount); + } + else + { + _logger.LogWarning("No items to cache for {Playlist}", playlistName); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to pre-build playlist items cache for {Playlist}", playlistName); + } + } + + /// + /// Saves playlist items to file cache for persistence across restarts. + /// + private async Task SavePlaylistItemsToFileAsync(string playlistName, List> items) + { + try + { + var cacheDir = "/app/cache/spotify"; + Directory.CreateDirectory(cacheDir); + + var safeName = string.Join("_", playlistName.Split(Path.GetInvalidFileNameChars())); + var filePath = Path.Combine(cacheDir, $"{safeName}_items.json"); + + var json = JsonSerializer.Serialize(items, new JsonSerializerOptions { WriteIndented = true }); + await System.IO.File.WriteAllTextAsync(filePath, json); + + _logger.LogDebug("💾 Saved {Count} playlist items to file cache: {Path}", items.Count, filePath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to save playlist items to file for {Playlist}", playlistName); + } + } } + diff --git a/allstarr/appsettings.json b/allstarr/appsettings.json index fdb962d..533f2ff 100644 --- a/allstarr/appsettings.json +++ b/allstarr/appsettings.json @@ -48,6 +48,7 @@ "SyncStartHour": 16, "SyncStartMinute": 15, "SyncWindowHours": 2, + "MatchingIntervalHours": 24, "Playlists": [] }, "SpotifyApi": {