Compare commits

...

10 Commits

Author SHA1 Message Date
6eaeee9a67 remove nonfunctional endpoint
Some checks failed
CI / build-and-test (push) Has been cancelled
2026-02-02 13:00:04 -05:00
9dd49a2f43 add round robin load balancing so providers dont hate me 2026-02-02 12:57:04 -05:00
2bc2816191 stupid timezones 2026-02-02 12:43:23 -05:00
1a2e160279 Update README pertaining to spotify playlist import 2026-02-02 12:32:11 -05:00
4111b5228d prematch on startup 2026-02-02 12:31:09 -05:00
82b480c47e remove useless md 2026-02-02 12:26:31 -05:00
0d246a8e74 gitignore update 2026-02-02 12:25:21 -05:00
fc3a8134ca spotify track prematching 2026-02-02 12:24:30 -05:00
2f91457e52 fix playlist, fix session sending 2026-02-02 12:18:29 -05:00
77774120bf playback start websocket 2026-02-02 12:09:29 -05:00
13 changed files with 851 additions and 205 deletions

View File

@@ -6,6 +6,10 @@ BACKEND_TYPE=Subsonic
# Enable Redis caching for metadata and images (default: true)
REDIS_ENABLED=true
# Redis data persistence directory (default: ./redis-data)
# Redis will save snapshots and append-only logs here to persist cache across restarts
REDIS_DATA_PATH=./redis-data
# ===== SUBSONIC/NAVIDROME CONFIGURATION =====
# Server URL (required if using Subsonic backend)
SUBSONIC_URL=http://localhost:4533

6
.gitignore vendored
View File

@@ -74,6 +74,12 @@ obj/
downloads/
!downloads/.gitkeep
# Kept music files (favorited external tracks)
kept/
# Cache files (Spotify missing tracks, etc.)
cache/
# Docker volumes
redis-data/

153
README.md
View File

@@ -250,6 +250,10 @@ Choose your preferred provider via the `MUSIC_SERVICE` environment variable. Add
|---------|-------------|
| `SquidWTF:Quality` | Preferred audio quality: `FLAC`, `MP3_320`, `MP3_128`. If not specified, the highest available quality for your account will be used |
**Load Balancing & Reliability:**
SquidWTF uses a round-robin load balancing strategy across multiple backup API endpoints to distribute requests evenly and prevent overwhelming any single provider. Each request automatically rotates to the next endpoint in the pool, with automatic fallback to other endpoints if one fails. This ensures high availability and prevents rate limiting by distributing load across multiple providers.
### Deezer Settings
| Setting | Description |
@@ -291,41 +295,148 @@ Subsonic__EnableExternalPlaylists=false
### Spotify Playlist Injection (Jellyfin Only)
Allstarr can intercept Spotify Import plugin playlists (Release Radar, Discover Weekly) and fill them with tracks automatically matched from your configured streaming provider (SquidWTF, Deezer, or Qobuz).
Allstarr can automatically fill your Spotify playlists (like Release Radar and Discover Weekly) with tracks from your configured streaming provider (SquidWTF, Deezer, or Qobuz). This feature works by intercepting playlists created by the Jellyfin Spotify Import plugin and matching missing tracks with your streaming service.
**Requirements:**
- [Jellyfin Spotify Import Plugin](https://github.com/Viperinius/jellyfin-plugin-spotify-import) installed and configured
- Plugin must run on a daily schedule (e.g., 4:15 PM daily)
- Jellyfin URL and API key configured (uses existing JELLYFIN_URL and JELLYFIN_API_KEY settings)
#### Prerequisites
**Configuration:**
1. **Install the Jellyfin Spotify Import Plugin**
- Navigate to Jellyfin Dashboard → Plugins → Catalog
- Search for "Spotify Import" by Viperinius
- Install and restart Jellyfin
- Plugin repository: [Viperinius/jellyfin-plugin-spotify-import](https://github.com/Viperinius/jellyfin-plugin-spotify-import)
2. **Configure the Spotify Import Plugin**
- Go to Jellyfin Dashboard → Plugins → Spotify Import
- Connect your Spotify account
- Select which playlists to sync (e.g., Release Radar, Discover Weekly)
- Set a daily sync schedule (e.g., 4:15 PM daily)
- The plugin will create playlists in Jellyfin and generate "missing tracks" files for songs not in your library
3. **Configure Allstarr**
- Allstarr needs to know when the plugin runs and which playlists to intercept
- Uses your existing `JELLYFIN_URL` and `JELLYFIN_API_KEY` settings (no additional credentials needed)
#### Configuration
| Setting | Description |
|---------|-------------|
| `SpotifyImport:Enabled` | Enable Spotify playlist injection (default: `false`) |
| `SpotifyImport:SyncStartHour` | Hour when plugin runs (24-hour format, 0-23) |
| `SpotifyImport:SyncStartMinute` | Minute when plugin runs (0-59) |
| `SpotifyImport:SyncWindowHours` | Hours to search for missing tracks files after sync time |
| `SpotifyImport:Playlists` | Array of playlists to inject (Name, SpotifyName, Enabled) |
| `SpotifyImport:SyncStartHour` | Hour when the Spotify Import plugin runs (24-hour format, 0-23) |
| `SpotifyImport:SyncStartMinute` | Minute when the plugin runs (0-59) |
| `SpotifyImport:SyncWindowHours` | Hours to search for missing tracks files after sync time (default: 2) |
| `SpotifyImport:PlaylistIds` | Comma-separated Jellyfin playlist IDs to intercept |
| `SpotifyImport:PlaylistNames` | Comma-separated playlist names (must match order of IDs) |
**How it works:**
1. Jellyfin Spotify Import plugin runs daily and creates playlists + missing tracks files
2. Allstarr fetches these missing tracks files within the configured time window
3. For each missing track, Allstarr searches your streaming provider (SquidWTF, Deezer, or Qobuz)
4. When you open the playlist in Jellyfin, Allstarr intercepts the request and returns matched tracks
5. Tracks are downloaded on-demand when played
6. On startup, Allstarr will fetch missing tracks if it hasn't run in the last 24 hours
**Environment variables:**
**Environment variables example:**
```bash
# Enable the feature
SPOTIFY_IMPORT_ENABLED=true
# Sync window settings (optional - used to prevent fetching too frequently)
# The fetcher searches backwards from current time for the last 48 hours
SPOTIFY_IMPORT_SYNC_START_HOUR=16
SPOTIFY_IMPORT_SYNC_START_MINUTE=15
SPOTIFY_IMPORT_SYNC_WINDOW_HOURS=2
SPOTIFY_IMPORT_PLAYLISTS=Release Radar,Discover Weekly
# Get playlist IDs from Jellyfin URLs: https://jellyfin.example.com/web/#/details?id=PLAYLIST_ID
SPOTIFY_IMPORT_PLAYLIST_IDS=ba50e26c867ec9d57ab2f7bf24cfd6b0,4383a46d8bcac3be2ef9385053ea18df
# Names must match exactly as they appear in Jellyfin (used to find missing tracks files)
SPOTIFY_IMPORT_PLAYLIST_NAMES=Release Radar,Discover Weekly
```
> **Note**: This feature uses your existing JELLYFIN_URL and JELLYFIN_API_KEY settings. The plugin must be configured to run on a schedule, and the sync window should cover the plugin's execution time.
#### How It Works
1. **Spotify Import Plugin Runs** (e.g., daily at 4:15 PM)
- Plugin fetches your Spotify playlists
- Creates/updates playlists in Jellyfin with tracks already in your library
- Generates "missing tracks" JSON files for songs not found locally
- Files are named like: `Release Radar_missing_2026-02-01_16-15.json`
2. **Allstarr Fetches Missing Tracks** (within sync window)
- Searches for missing tracks files from the Jellyfin plugin
- Searches **+24 hours forward first** (newest files), then **-48 hours backward** if not found
- This efficiently finds the most recent file regardless of timezone differences
- Example: Server time 12 PM EST, file timestamped 9 PM UTC (same day) → Found in forward search
- Caches the list of missing tracks in Redis + file cache
- Runs automatically on startup (if needed) and every 5 minutes during the sync window
3. **Allstarr Matches Tracks** (2 minutes after startup, then every 30 minutes)
- For each missing track, searches your streaming provider (SquidWTF, Deezer, or Qobuz)
- Uses fuzzy matching to find the best match (title + artist similarity)
- Rate-limited to avoid overwhelming the service (150ms delay between searches)
- Caches matched results for 1 hour
4. **You Open the Playlist in Jellyfin**
- Allstarr intercepts the request
- Returns a merged list: local tracks + matched streaming tracks
- Loads instantly from cache (no searching needed!)
5. **You Play a Track**
- If it's a local track, streams from Jellyfin normally
- If it's a matched track, downloads from streaming provider on-demand
- Downloaded tracks are saved to your library for future use
#### Manual Triggers
You can manually trigger syncing and matching via API:
```bash
# Fetch missing tracks from Jellyfin plugin
curl "https://your-jellyfin-proxy.com/spotify/sync?api_key=YOUR_API_KEY"
# Trigger track matching (searches streaming provider)
curl "https://your-jellyfin-proxy.com/spotify/match?api_key=YOUR_API_KEY"
# Clear cache to force re-matching
curl "https://your-jellyfin-proxy.com/spotify/clear-cache?api_key=YOUR_API_KEY"
```
#### Startup Behavior
When Allstarr starts with Spotify Import enabled:
**Smart Cache Check:**
- Checks if today's sync window has passed (e.g., if sync is at 4 PM + 2 hour window = 6 PM)
- If before 6 PM and yesterday's cache exists → **Skips fetch** (cache is still current)
- If after 6 PM or no cache exists → **Fetches missing tracks** from Jellyfin plugin
**Track Matching:**
- **T+2min**: Matches tracks with streaming provider (with rate limiting)
- Only matches playlists that don't already have cached matches
- **Result**: Playlists load instantly when you open them!
**Example Timeline:**
- Plugin runs daily at 4:15 PM, creates files at ~4:16 PM
- You restart Allstarr at 12:00 PM (noon) the next day
- Startup check: "Today's sync window ends at 6 PM, and I have yesterday's 4:16 PM file"
- **Decision**: Skip fetch, use existing cache
- At 6:01 PM: Next scheduled check will search for new files
#### Troubleshooting
**Playlists are empty:**
- Check that the Spotify Import plugin is running and creating playlists
- Verify `SPOTIFY_IMPORT_PLAYLIST_IDS` match your Jellyfin playlist IDs
- Check logs: `docker-compose logs -f allstarr | grep -i spotify`
**Tracks aren't matching:**
- Ensure your streaming provider is configured (`MUSIC_SERVICE`, credentials)
- Check that playlist names in `SPOTIFY_IMPORT_PLAYLIST_NAMES` match exactly
- Manually trigger matching: `curl "https://your-proxy.com/spotify/match?api_key=KEY"`
**Sync timing issues:**
- Set `SPOTIFY_IMPORT_SYNC_START_HOUR/MINUTE` to match your plugin schedule
- Increase `SPOTIFY_IMPORT_SYNC_WINDOW_HOURS` if files aren't being found
- Check Jellyfin plugin logs to confirm when it runs
#### Notes
- This feature uses your existing `JELLYFIN_URL` and `JELLYFIN_API_KEY` settings
- Matched tracks are cached for 1 hour to avoid repeated searches
- Missing tracks cache persists across restarts (stored in Redis + file cache)
- Rate limiting prevents overwhelming your streaming provider (150ms between searches)
- Only works with Jellyfin backend (not Subsonic/Navidrome)
### Getting Credentials

View File

@@ -63,11 +63,12 @@ public class JellyfinProxyServiceTests
SetupMockResponse(HttpStatusCode.OK, jsonResponse, "application/json");
// Act
var result = await _service.GetJsonAsync("Items");
var (body, statusCode) = await _service.GetJsonAsync("Items");
// Assert
Assert.NotNull(result);
Assert.True(result.RootElement.TryGetProperty("Items", out var items));
Assert.NotNull(body);
Assert.Equal(200, statusCode);
Assert.True(body.RootElement.TryGetProperty("Items", out var items));
Assert.Equal(1, items.GetArrayLength());
}
@@ -78,10 +79,11 @@ public class JellyfinProxyServiceTests
SetupMockResponse(HttpStatusCode.InternalServerError, "", "text/plain");
// Act
var result = await _service.GetJsonAsync("Items");
var (body, statusCode) = await _service.GetJsonAsync("Items");
// Assert
Assert.Null(result);
Assert.Null(body);
Assert.Equal(500, statusCode);
}
[Fact]
@@ -207,12 +209,13 @@ public class JellyfinProxyServiceTests
});
// Act
var result = await _service.GetItemAsync("abc-123");
var (body, statusCode) = await _service.GetItemAsync("abc-123");
// Assert
Assert.NotNull(captured);
Assert.Contains("/Items/abc-123", captured!.RequestUri!.ToString());
Assert.NotNull(result);
Assert.NotNull(body);
Assert.Equal(200, statusCode);
}
[Fact]

View File

@@ -1903,15 +1903,60 @@ public class JellyfinController : ControllerBase
// For local tracks, forward to Jellyfin with client auth
_logger.LogInformation("Forwarding playback start to Jellyfin...");
var (result, statusCode) = await _proxyService.PostJsonAsync("Sessions/Playing", body, Request.Headers);
// Fetch full item details to include in playback report
// This makes the session show up properly in Jellyfin dashboard with "Now Playing"
try
{
var (itemResult, itemStatus) = await _proxyService.GetJsonAsync($"Items/{itemId}", null, Request.Headers);
if (itemResult != null && itemStatus == 200)
{
var item = itemResult.RootElement;
_logger.LogInformation("📦 Fetched item details for playback report");
// Build enhanced playback start info with full item details
var enhancedBody = new
{
ItemId = itemId,
PositionTicks = doc.RootElement.TryGetProperty("PositionTicks", out var posProp) ? posProp.GetInt64() : 0,
// Include the full item so Jellyfin can display "Now Playing"
NowPlayingItem = item.Clone()
};
var enhancedJson = JsonSerializer.Serialize(enhancedBody);
_logger.LogInformation("📤 Sending enhanced playback start with item details");
var (result, statusCode) = await _proxyService.PostJsonAsync("Sessions/Playing", enhancedJson, Request.Headers);
if (statusCode == 204 || statusCode == 200)
{
_logger.LogInformation("✓ Playback start forwarded to Jellyfin ({StatusCode})", statusCode);
_logger.LogInformation("✓ Enhanced playback start forwarded to Jellyfin ({StatusCode})", statusCode);
}
else
{
_logger.LogWarning("⚠️ Playback start forward returned status {StatusCode}", statusCode);
_logger.LogWarning("⚠️ Enhanced playback start returned status {StatusCode}", statusCode);
}
}
else
{
_logger.LogWarning("⚠️ Could not fetch item details ({StatusCode}), sending basic playback start", itemStatus);
// Fall back to basic playback start
var (result, statusCode) = await _proxyService.PostJsonAsync("Sessions/Playing", body, Request.Headers);
if (statusCode == 204 || statusCode == 200)
{
_logger.LogInformation("✓ Basic playback start forwarded to Jellyfin ({StatusCode})", statusCode);
}
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to send enhanced playback start, trying basic");
// Fall back to basic playback start
var (result, statusCode) = await _proxyService.PostJsonAsync("Sessions/Playing", body, Request.Headers);
if (statusCode == 204 || statusCode == 200)
{
_logger.LogInformation("✓ Basic playback start forwarded to Jellyfin ({StatusCode})", statusCode);
}
}
return NoContent();
@@ -2403,7 +2448,10 @@ public class JellyfinController : ControllerBase
}
// Modify response if it contains Spotify playlists to update ChildCount
if (_spotifySettings.Enabled && result.RootElement.TryGetProperty("Items", out var items))
// Only check for Items if the response is an object (not a string or array)
if (_spotifySettings.Enabled &&
result.RootElement.ValueKind == JsonValueKind.Object &&
result.RootElement.TryGetProperty("Items", out var items))
{
_logger.LogInformation("Response has Items property, checking for Spotify playlists to update counts");
result = await UpdateSpotifyPlaylistCounts(result);
@@ -2737,11 +2785,11 @@ public class JellyfinController : ControllerBase
{
missingTracks = await LoadMissingTracksFromFile(spotifyPlaylistName);
// If we loaded from file, restore to Redis
// If we loaded from file, restore to Redis with no expiration
if (missingTracks != null && missingTracks.Count > 0)
{
await _cache.SetAsync(missingTracksKey, missingTracks, TimeSpan.FromHours(24));
_logger.LogInformation("Restored {Count} missing tracks from file cache for {Playlist}",
await _cache.SetAsync(missingTracksKey, missingTracks, TimeSpan.FromDays(365));
_logger.LogInformation("Restored {Count} missing tracks from file cache for {Playlist} (no expiration)",
missingTracks.Count, spotifyPlaylistName);
}
}
@@ -2756,10 +2804,13 @@ public class JellyfinController : ControllerBase
_logger.LogInformation("Matching {Count} missing tracks for {Playlist}",
missingTracks.Count, spotifyPlaylistName);
// Match missing tracks (excluding ones we already have locally)
var matchTasks = missingTracks
// Match missing tracks sequentially with rate limiting (excluding ones we already have locally)
var matchedBySpotifyId = new Dictionary<string, Song>();
var tracksToMatch = missingTracks
.Where(track => !existingSpotifyIds.Contains(track.SpotifyId))
.Select(async track =>
.ToList();
foreach (var track in tracksToMatch)
{
try
{
@@ -2767,17 +2818,17 @@ public class JellyfinController : ControllerBase
var query = $"{track.Title} {track.PrimaryArtist}";
var results = await _metadataService.SearchSongsAsync(query, limit: 5);
if (results.Count == 0)
return (track.SpotifyId, (Song?)null);
if (results.Count > 0)
{
// Fuzzy match to find best result
// Check that ALL artists match (not just some)
var bestMatch = results
.Select(song => new
{
Song = song,
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist),
TotalScore = 0.0
// Calculate artist score by checking ALL artists match
ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
})
.Select(x => new
{
@@ -2789,32 +2840,31 @@ public class JellyfinController : ControllerBase
.OrderByDescending(x => x.TotalScore)
.FirstOrDefault();
// Only return if match is good enough (>60% combined score)
// Only add if match is good enough (>60% combined score)
if (bestMatch != null && bestMatch.TotalScore >= 60)
{
_logger.LogDebug("Matched '{Title}' by {Artist} -> '{MatchTitle}' by {MatchArtist} (score: {Score:F1})",
track.Title, track.PrimaryArtist,
bestMatch.Song.Title, bestMatch.Song.Artist,
bestMatch.TotalScore);
return (track.SpotifyId, (Song?)bestMatch.Song);
matchedBySpotifyId[track.SpotifyId] = bestMatch.Song;
}
else
{
_logger.LogDebug("No good match for '{Title}' by {Artist} (best score: {Score:F1})",
track.Title, track.PrimaryArtist, bestMatch?.TotalScore ?? 0);
return (track.SpotifyId, (Song?)null);
}
}
// Rate limiting: small delay between searches to avoid overwhelming the service
await Task.Delay(100); // 100ms delay = max 10 searches/second
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}",
track.Title, track.PrimaryArtist);
return (track.SpotifyId, (Song?)null);
}
});
var matchResults = await Task.WhenAll(matchTasks);
var matchedBySpotifyId = matchResults
.Where(x => x.Item2 != null)
.ToDictionary(x => x.SpotifyId, x => x.Item2!);
}
// Build final track list in Spotify playlist order
var finalTracks = new List<Song>();
@@ -2936,12 +2986,9 @@ public class JellyfinController : ControllerBase
return null;
}
// No expiration check - cache persists until next Jellyfin job generates new file
var fileAge = DateTime.UtcNow - System.IO.File.GetLastWriteTimeUtc(filePath);
if (fileAge > TimeSpan.FromHours(24))
{
_logger.LogDebug("File cache for {Playlist} is too old ({Age:F1}h)", playlistName, fileAge.TotalHours);
return null;
}
_logger.LogDebug("File cache for {Playlist} age: {Age:F1}h (no expiration)", playlistName, fileAge.TotalHours);
var json = await System.IO.File.ReadAllTextAsync(filePath);
var tracks = JsonSerializer.Deserialize<List<allstarr.Models.Spotify.MissingTrack>>(json);
@@ -3050,6 +3097,54 @@ public class JellyfinController : ControllerBase
return Ok(results);
}
/// <summary>
/// Manually trigger track matching for all Spotify playlists.
/// GET /spotify/match?api_key=YOUR_KEY
/// </summary>
[HttpGet("spotify/match", Order = 1)]
[ServiceFilter(typeof(ApiKeyAuthFilter))]
public async Task<IActionResult> TriggerSpotifyMatch([FromServices] IEnumerable<IHostedService> hostedServices)
{
if (!_spotifySettings.Enabled)
{
return BadRequest(new { error = "Spotify Import is not enabled" });
}
_logger.LogInformation("Manual Spotify track matching triggered");
// Find the SpotifyTrackMatchingService
var matchingService = hostedServices
.OfType<allstarr.Services.Spotify.SpotifyTrackMatchingService>()
.FirstOrDefault();
if (matchingService == null)
{
return StatusCode(500, new { error = "SpotifyTrackMatchingService not found" });
}
// Trigger matching asynchronously
_ = Task.Run(async () =>
{
try
{
await matchingService.TriggerMatchingAsync();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during manual track matching");
}
});
return Ok(new
{
status = "started",
message = "Track matching started in background. Check logs for progress.",
playlists = _spotifySettings.PlaylistNames.Count > 0
? _spotifySettings.PlaylistNames
: _spotifySettings.PlaylistIds
});
}
private List<allstarr.Models.Spotify.MissingTrack> ParseMissingTracksJson(string json)
{
var tracks = new List<allstarr.Models.Spotify.MissingTrack>();
@@ -3226,5 +3321,53 @@ public class JellyfinController : ControllerBase
}
#endregion
/// <summary>
/// Calculates artist match score ensuring ALL artists are present.
/// Penalizes if artist counts don't match or if any artist is missing.
/// </summary>
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
{
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
return 0;
// Build list of all song artists (main + contributors)
var allSongArtists = new List<string> { songMainArtist };
allSongArtists.AddRange(songContributors);
// If artist counts differ significantly, penalize
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
return 0;
// Check that each Spotify artist has a good match in song artists
var spotifyScores = new List<double>();
foreach (var spotifyArtist in spotifyArtists)
{
var bestMatch = allSongArtists.Max(songArtist =>
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
spotifyScores.Add(bestMatch);
}
// Check that each song artist has a good match in Spotify artists
var songScores = new List<double>();
foreach (var songArtist in allSongArtists)
{
var bestMatch = spotifyArtists.Max(spotifyArtist =>
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
songScores.Add(bestMatch);
}
// Average all scores - this ensures ALL artists must match well
var allScores = spotifyScores.Concat(songScores);
var avgScore = allScores.Average();
// Penalize if any individual artist match is poor (< 70)
var minScore = allScores.Min();
if (minScore < 70)
avgScore *= 0.7; // 30% penalty for poor individual match
return avgScore;
}
}
// force rebuild Sun Jan 25 13:22:47 EST 2026

View File

@@ -14,19 +14,22 @@ public class SpotifyImportSettings
/// <summary>
/// Hour when Spotify Import plugin runs (24-hour format, 0-23)
/// Example: 16 for 4:00 PM
/// NOTE: This setting is now optional and only used for the sync window check.
/// The fetcher will search backwards from current time for the last 48 hours,
/// so timezone confusion is avoided.
/// </summary>
public int SyncStartHour { get; set; } = 16;
/// <summary>
/// Minute when Spotify Import plugin runs (0-59)
/// Example: 15 for 4:15 PM
/// NOTE: This setting is now optional and only used for the sync window check.
/// </summary>
public int SyncStartMinute { get; set; } = 15;
/// <summary>
/// How many hours to search for missing tracks files after sync start time
/// Example: 2 means search from 4:00 PM to 6:00 PM
/// This prevents the fetcher from running too frequently.
/// Set to 0 to disable the sync window check and always search on startup.
/// </summary>
public int SyncWindowHours { get; set; } = 2;

View File

@@ -23,11 +23,15 @@ static List<string> DecodeSquidWtfUrls()
var encodedUrls = new[]
{
"aHR0cHM6Ly90cml0b24uc3F1aWQud3Rm", // triton
"aHR0cHM6Ly90aWRhbC1hcGkuYmluaW11bS5vcmc=", // binimum
"aHR0cHM6Ly90aWRhbC5raW5vcGx1cy5vbmxpbmU=", // kinoplus
"aHR0cHM6Ly9oaWZpLXR3by5zcG90aXNhdmVyLm5ldA==", // spoti-2
"aHR0cHM6Ly9oaWZpLW9uZS5zcG90aXNhdmVyLm5ldA==", // spoti-1
"aHR0cHM6Ly93b2xmLnFxZGwuc2l0ZQ==", // wolf
"aHR0cDovL2h1bmQucXFkbC5zaXRl", // hund
"aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==", // maus
"aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=", // katze
"aHR0cHM6Ly92b2dlbC5xcWRsLnNpdGU=", // vogel
"aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=" // katze
"aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==" // maus
};
return encodedUrls
@@ -269,6 +273,9 @@ builder.Services.AddHostedService<CacheCleanupService>();
// Register Spotify missing tracks fetcher (only runs when SpotifyImport is enabled)
builder.Services.AddHostedService<allstarr.Services.Spotify.SpotifyMissingTracksFetcher>();
// Register Spotify track matching service (pre-matches tracks with rate limiting)
builder.Services.AddHostedService<allstarr.Services.Spotify.SpotifyTrackMatchingService>();
builder.Services.AddCors(options =>
{
options.AddDefaultPolicy(policy =>

View File

@@ -72,13 +72,13 @@ public class SpotifyMissingTracksFetcher : BackgroundService
}
_logger.LogInformation("========================================");
// Run once on startup if we haven't run in the last 24 hours
// Check if we should run on startup
if (!_hasRunOnce)
{
var shouldRunOnStartup = await ShouldRunOnStartupAsync();
if (shouldRunOnStartup)
var shouldRun = await ShouldRunOnStartupAsync();
if (shouldRun)
{
_logger.LogInformation("Running initial fetch on startup (bypassing sync window check)");
_logger.LogInformation("Running initial fetch on startup");
try
{
await FetchMissingTracksAsync(stoppingToken, bypassSyncWindowCheck: true);
@@ -91,7 +91,7 @@ public class SpotifyMissingTracksFetcher : BackgroundService
}
else
{
_logger.LogInformation("Skipping startup fetch - already have recent cache");
_logger.LogInformation("Skipping startup fetch - existing cache is still current");
_hasRunOnce = true;
}
}
@@ -130,86 +130,69 @@ public class SpotifyMissingTracksFetcher : BackgroundService
private async Task<bool> ShouldRunOnStartupAsync()
{
_logger.LogInformation("=== STARTUP CACHE CHECK ===");
_logger.LogInformation("Cache directory: {Dir}", CacheDirectory);
_logger.LogInformation("Checking {Count} playlists", _playlistIdToName.Count);
// List all files in cache directory for debugging
try
{
if (Directory.Exists(CacheDirectory))
{
var files = Directory.GetFiles(CacheDirectory, "*.json");
_logger.LogInformation("Found {Count} JSON files in cache directory:", files.Length);
foreach (var file in files)
{
var fileInfo = new FileInfo(file);
var age = DateTime.UtcNow - fileInfo.LastWriteTimeUtc;
_logger.LogInformation(" - {Name} (age: {Age:F1}h, size: {Size} bytes)",
Path.GetFileName(file), age.TotalHours, fileInfo.Length);
}
}
else
{
_logger.LogWarning("Cache directory does not exist: {Dir}", CacheDirectory);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error listing cache directory");
}
var settings = _spotifySettings.Value;
var now = DateTime.UtcNow;
// Calculate when today's sync window ends
var todaySync = now.Date
.AddHours(settings.SyncStartHour)
.AddMinutes(settings.SyncStartMinute);
var todaySyncEnd = todaySync.AddHours(settings.SyncWindowHours);
// If we haven't reached today's sync window end yet, check if we have yesterday's file
if (now < todaySyncEnd)
{
_logger.LogInformation("Today's sync window hasn't ended yet (ends at {End})", todaySyncEnd);
_logger.LogInformation("Checking if we have a recent cache file...");
// Check if we have any cache (file or Redis) for all playlists
var allPlaylistsHaveCache = true;
// Check file cache first, then Redis
foreach (var playlistName in _playlistIdToName.Values)
{
var filePath = GetCacheFilePath(playlistName);
_logger.LogInformation("Checking playlist: {Playlist}", playlistName);
_logger.LogInformation(" Expected file path: {Path}", filePath);
var cacheKey = $"spotify:missing:{playlistName}";
// Check file cache
if (File.Exists(filePath))
{
var fileAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(filePath);
_logger.LogInformation(" File exists! Age: {Age:F1}h", fileAge.TotalHours);
_logger.LogInformation(" {Playlist}: Found file cache (age: {Age:F1}h)", playlistName, fileAge.TotalHours);
if (fileAge < TimeSpan.FromHours(24))
// Load into Redis if not already there
if (!await _cache.ExistsAsync(cacheKey))
{
_logger.LogInformation(" ✓ Found recent file cache (age: {Age:F1}h)", fileAge.TotalHours);
// Load from file into Redis if not already there
var key = $"spotify:missing:{playlistName}";
if (!await _cache.ExistsAsync(key))
{
_logger.LogInformation(" Loading into Redis...");
await LoadFromFileCache(playlistName);
}
else
{
_logger.LogInformation(" Already in Redis");
}
return false;
}
else
{
_logger.LogInformation(" File too old ({Age:F1}h > 24h), will fetch new", fileAge.TotalHours);
}
}
else
{
_logger.LogInformation(" File does not exist at expected path");
continue;
}
var cacheKey = $"spotify:missing:{playlistName}";
// Check Redis cache
if (await _cache.ExistsAsync(cacheKey))
{
_logger.LogInformation(" Found in Redis cache");
_logger.LogInformation(" {Playlist}: Found in Redis cache", playlistName);
continue;
}
// No cache found for this playlist
_logger.LogInformation(" {Playlist}: No cache found", playlistName);
allPlaylistsHaveCache = false;
}
if (allPlaylistsHaveCache)
{
_logger.LogInformation("=== ALL PLAYLISTS HAVE CACHE - SKIPPING STARTUP FETCH ===");
return false;
}
}
else
{
_logger.LogInformation(" Not in Redis cache");
}
_logger.LogInformation("Today's sync window has passed (ended at {End})", todaySyncEnd);
_logger.LogInformation("Will search for new files");
}
_logger.LogInformation("=== NO RECENT CACHE FOUND - WILL FETCH ===");
_logger.LogInformation("=== WILL FETCH ON STARTUP ===");
return true;
}
@@ -234,14 +217,11 @@ public class SpotifyMissingTracksFetcher : BackgroundService
{
var cacheKey = $"spotify:missing:{playlistName}";
var fileAge = DateTime.UtcNow - File.GetLastWriteTimeUtc(filePath);
var ttl = TimeSpan.FromHours(24) - fileAge;
if (ttl > TimeSpan.Zero)
{
await _cache.SetAsync(cacheKey, tracks, ttl);
_logger.LogInformation("Loaded {Count} tracks from file cache for {Playlist}",
tracks.Count, playlistName);
}
// No expiration - cache persists until next Jellyfin job generates new file
await _cache.SetAsync(cacheKey, tracks, TimeSpan.FromDays(365));
_logger.LogInformation("Loaded {Count} tracks from file cache for {Playlist} (age: {Age:F1}h, no expiration)",
tracks.Count, playlistName, fileAge.TotalHours);
}
}
catch (Exception ex)
@@ -308,13 +288,26 @@ public class SpotifyMissingTracksFetcher : BackgroundService
{
var cacheKey = $"spotify:missing:{playlistName}";
if (await _cache.ExistsAsync(cacheKey))
// Check if we have existing cache and when it was last updated
var existingTracks = await _cache.GetAsync<List<MissingTrack>>(cacheKey);
var existingFileTime = DateTime.MinValue;
var filePath = GetCacheFilePath(playlistName);
if (File.Exists(filePath))
{
_logger.LogInformation(" ✓ Cache already exists for {Playlist}, skipping fetch", playlistName);
return;
existingFileTime = File.GetLastWriteTimeUtc(filePath);
_logger.LogInformation(" Existing cache file from: {Time} ({Age:F1}h ago)",
existingFileTime, (DateTime.UtcNow - existingFileTime).TotalHours);
}
_logger.LogInformation(" No cache found, will search for missing tracks file...");
if (existingTracks != null && existingTracks.Count > 0)
{
_logger.LogInformation(" Current cache has {Count} tracks, will search for newer file", existingTracks.Count);
}
else
{
_logger.LogInformation(" No existing cache, will search for missing tracks file");
}
var settings = _spotifySettings.Value;
var jellyfinUrl = _jellyfinSettings.Value.Url;
@@ -328,75 +321,121 @@ public class SpotifyMissingTracksFetcher : BackgroundService
var httpClient = _httpClientFactory.CreateClient();
// Start from the configured sync time (most likely time)
// Search forward first (newest files), then backwards to handle timezone differences
// We want the file with the furthest forward timestamp (most recent)
var now = DateTime.UtcNow;
var todaySync = now.Date
.AddHours(settings.SyncStartHour)
.AddMinutes(settings.SyncStartMinute);
// If we haven't reached today's sync time yet, start from yesterday's sync time
var syncTime = now >= todaySync ? todaySync : todaySync.AddDays(-1);
_logger.LogInformation(" Searching +12h forward, -24h backward from {SyncTime}", syncTime);
_logger.LogInformation(" Searching +24h forward, then -48h backward from {Now}", now);
var found = false;
DateTime? foundFileTime = null;
// Search forward 12 hours from sync time
_logger.LogInformation(" Phase 1: Searching forward 12 hours from sync time...");
for (var minutesAhead = 0; minutesAhead <= 720; minutesAhead++) // 720 minutes = 12 hours
// First search forward 24 hours (most likely to find newest files with timezone ahead)
_logger.LogInformation(" Phase 1: Searching forward 24 hours...");
for (var minutesAhead = 1; minutesAhead <= 1440; minutesAhead++)
{
if (cancellationToken.IsCancellationRequested) break;
var time = syncTime.AddMinutes(minutesAhead);
if (await TryFetchMissingTracksFile(playlistName, time, jellyfinUrl, apiKey, httpClient, cancellationToken))
var time = now.AddMinutes(minutesAhead);
var result = await TryFetchMissingTracksFile(playlistName, time, jellyfinUrl, apiKey, httpClient, cancellationToken, existingFileTime);
if (result.found)
{
found = true;
break;
}
// Small delay every 60 requests
if (minutesAhead > 0 && minutesAhead % 60 == 0)
foundFileTime = result.fileTime;
if (foundFileTime.HasValue)
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
_logger.LogInformation(" ✓ Found file from {Time} (+{Offset:F1}h ahead)",
foundFileTime.Value, (foundFileTime.Value - now).TotalHours);
}
break; // Found newest file, stop searching
}
// Small delay every 60 requests to avoid rate limiting
if (minutesAhead % 60 == 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
}
}
// Then search backwards 24 hours from sync time to catch yesterday's file
// If not found forward, search backwards 48 hours
if (!found)
{
_logger.LogInformation(" Phase 2: Searching backward 24 hours from sync time...");
for (var minutesBehind = 1; minutesBehind <= 1440; minutesBehind++) // 1440 minutes = 24 hours
_logger.LogInformation(" Phase 2: Searching backward 48 hours...");
for (var minutesBehind = 0; minutesBehind <= 2880; minutesBehind++)
{
if (cancellationToken.IsCancellationRequested) break;
var time = syncTime.AddMinutes(-minutesBehind);
if (await TryFetchMissingTracksFile(playlistName, time, jellyfinUrl, apiKey, httpClient, cancellationToken))
var time = now.AddMinutes(-minutesBehind);
var result = await TryFetchMissingTracksFile(playlistName, time, jellyfinUrl, apiKey, httpClient, cancellationToken, existingFileTime);
if (result.found)
{
found = true;
foundFileTime = result.fileTime;
if (foundFileTime.HasValue)
{
_logger.LogInformation(" ✓ Found file from {Time} (-{Offset:F1}h ago)",
foundFileTime.Value, (now - foundFileTime.Value).TotalHours);
}
break;
}
// Small delay every 60 requests
if (minutesBehind % 60 == 0)
// Small delay every 60 requests to avoid rate limiting
if (minutesBehind > 0 && minutesBehind % 60 == 0)
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
}
}
}
if (!found)
{
_logger.LogWarning(" ✗ Could not find missing tracks file (searched +12h/-24h window)");
_logger.LogWarning(" ✗ Could not find new missing tracks file (searched +24h forward, -48h backward)");
// Keep the existing cache - don't let it expire
if (existingTracks != null && existingTracks.Count > 0)
{
_logger.LogInformation(" ✓ Keeping existing cache with {Count} tracks (no expiration)", existingTracks.Count);
// Re-save with no expiration to ensure it persists
await _cache.SetAsync(cacheKey, existingTracks, TimeSpan.FromDays(365)); // Effectively no expiration
}
else if (File.Exists(filePath))
{
// Load from file if Redis cache is empty
_logger.LogInformation(" 📦 Loading existing file cache to keep playlist populated");
try
{
var json = await File.ReadAllTextAsync(filePath, cancellationToken);
var tracks = JsonSerializer.Deserialize<List<MissingTrack>>(json);
if (tracks != null && tracks.Count > 0)
{
await _cache.SetAsync(cacheKey, tracks, TimeSpan.FromDays(365)); // No expiration
_logger.LogInformation(" ✓ Loaded {Count} tracks from file cache (no expiration)", tracks.Count);
}
}
catch (Exception ex)
{
_logger.LogError(ex, " Failed to reload cache from file for {Playlist}", playlistName);
}
}
else
{
_logger.LogWarning(" No existing cache to keep - playlist will be empty until tracks are found");
}
}
else if (foundFileTime.HasValue)
{
_logger.LogInformation(" ✓ Updated cache with newer file from {Time}", foundFileTime.Value);
}
}
private async Task<bool> TryFetchMissingTracksFile(
private async Task<(bool found, DateTime? fileTime)> TryFetchMissingTracksFile(
string playlistName,
DateTime time,
string jellyfinUrl,
string apiKey,
HttpClient httpClient,
CancellationToken cancellationToken)
CancellationToken cancellationToken,
DateTime existingFileTime)
{
var filename = $"{playlistName}_missing_{time:yyyy-MM-dd_HH-mm}.json";
var url = $"{jellyfinUrl}/Viperinius.Plugin.SpotifyImport/MissingTracksFile" +
@@ -413,16 +452,24 @@ public class SpotifyMissingTracksFetcher : BackgroundService
if (tracks.Count > 0)
{
// Check if this file is newer than what we already have
if (time <= existingFileTime)
{
_logger.LogDebug(" Skipping {Filename} - not newer than existing cache", filename);
return (false, null);
}
var cacheKey = $"spotify:missing:{playlistName}";
// Save to both Redis and file
await _cache.SetAsync(cacheKey, tracks, TimeSpan.FromHours(24));
// Save to both Redis and file with extended TTL until next job runs
// Set to 365 days (effectively no expiration) - will be replaced when Jellyfin generates new file
await _cache.SetAsync(cacheKey, tracks, TimeSpan.FromDays(365));
await SaveToFileCache(playlistName, tracks);
_logger.LogInformation(
"✓ Cached {Count} missing tracks for {Playlist} from {Filename}",
"✓ Cached {Count} missing tracks for {Playlist} from {Filename} (no expiration until next Jellyfin job)",
tracks.Count, playlistName, filename);
return true;
return (true, time);
}
}
}
@@ -431,7 +478,7 @@ public class SpotifyMissingTracksFetcher : BackgroundService
_logger.LogDebug(ex, "Failed to fetch {Filename}", filename);
}
return false;
return (false, null);
}
private List<MissingTrack> ParseMissingTracks(string json)

View File

@@ -0,0 +1,259 @@
using allstarr.Models.Domain;
using allstarr.Models.Settings;
using allstarr.Models.Spotify;
using allstarr.Services.Common;
using Microsoft.Extensions.Options;
using System.Text.Json;
namespace allstarr.Services.Spotify;
/// <summary>
/// Background service that pre-matches Spotify missing tracks with external providers.
/// Runs after SpotifyMissingTracksFetcher completes to avoid rate limiting during playlist loading.
/// </summary>
public class SpotifyTrackMatchingService : BackgroundService
{
private readonly IOptions<SpotifyImportSettings> _spotifySettings;
private readonly RedisCacheService _cache;
private readonly ILogger<SpotifyTrackMatchingService> _logger;
private readonly IServiceProvider _serviceProvider;
private const int DelayBetweenSearchesMs = 150; // 150ms = ~6.6 searches/second to avoid rate limiting
public SpotifyTrackMatchingService(
IOptions<SpotifyImportSettings> spotifySettings,
RedisCacheService cache,
IServiceProvider serviceProvider,
ILogger<SpotifyTrackMatchingService> logger)
{
_spotifySettings = spotifySettings;
_cache = cache;
_serviceProvider = serviceProvider;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("SpotifyTrackMatchingService: Starting up...");
if (!_spotifySettings.Value.Enabled)
{
_logger.LogInformation("Spotify playlist injection is DISABLED, matching service will not run");
return;
}
// Wait a bit for the fetcher to run first
await Task.Delay(TimeSpan.FromMinutes(2), stoppingToken);
// Run once on startup to match any existing missing tracks
try
{
_logger.LogInformation("Running initial track matching on startup");
await MatchAllPlaylistsAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error during startup track matching");
}
while (!stoppingToken.IsCancellationRequested)
{
try
{
await MatchAllPlaylistsAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error in track matching service");
}
// Run every 30 minutes to catch new missing tracks
await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken);
}
}
/// <summary>
/// Public method to trigger matching manually (called from controller).
/// </summary>
public async Task TriggerMatchingAsync()
{
_logger.LogInformation("Manual track matching triggered");
await MatchAllPlaylistsAsync(CancellationToken.None);
}
private async Task MatchAllPlaylistsAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("=== STARTING TRACK MATCHING ===");
var playlistNames = _spotifySettings.Value.PlaylistNames;
if (playlistNames.Count == 0)
{
_logger.LogInformation("No playlists configured for matching");
return;
}
using var scope = _serviceProvider.CreateScope();
var metadataService = scope.ServiceProvider.GetRequiredService<IMusicMetadataService>();
foreach (var playlistName in playlistNames)
{
if (cancellationToken.IsCancellationRequested) break;
try
{
await MatchPlaylistTracksAsync(playlistName, metadataService, cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error matching tracks for playlist {Playlist}", playlistName);
}
}
_logger.LogInformation("=== FINISHED TRACK MATCHING ===");
}
private async Task MatchPlaylistTracksAsync(
string playlistName,
IMusicMetadataService metadataService,
CancellationToken cancellationToken)
{
var missingTracksKey = $"spotify:missing:{playlistName}";
var matchedTracksKey = $"spotify:matched:{playlistName}";
// Check if we already have matched tracks cached
var existingMatched = await _cache.GetAsync<List<Song>>(matchedTracksKey);
if (existingMatched != null && existingMatched.Count > 0)
{
_logger.LogInformation("Playlist {Playlist} already has {Count} matched tracks cached, skipping",
playlistName, existingMatched.Count);
return;
}
// Get missing tracks
var missingTracks = await _cache.GetAsync<List<MissingTrack>>(missingTracksKey);
if (missingTracks == null || missingTracks.Count == 0)
{
_logger.LogInformation("No missing tracks found for {Playlist}, skipping matching", playlistName);
return;
}
_logger.LogInformation("Matching {Count} tracks for {Playlist} (with rate limiting)",
missingTracks.Count, playlistName);
var matchedSongs = new List<Song>();
var matchCount = 0;
foreach (var track in missingTracks)
{
if (cancellationToken.IsCancellationRequested) break;
try
{
var query = $"{track.Title} {track.PrimaryArtist}";
var results = await metadataService.SearchSongsAsync(query, limit: 5);
if (results.Count > 0)
{
// Fuzzy match to find best result
// Check that ALL artists match (not just some)
var bestMatch = results
.Select(song => new
{
Song = song,
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
// Calculate artist score by checking ALL artists match
ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
})
.Select(x => new
{
x.Song,
x.TitleScore,
x.ArtistScore,
TotalScore = (x.TitleScore * 0.6) + (x.ArtistScore * 0.4)
})
.OrderByDescending(x => x.TotalScore)
.FirstOrDefault();
if (bestMatch != null && bestMatch.TotalScore >= 60)
{
matchedSongs.Add(bestMatch.Song);
matchCount++;
if (matchCount % 10 == 0)
{
_logger.LogInformation("Matched {Count}/{Total} tracks for {Playlist}",
matchCount, missingTracks.Count, playlistName);
}
}
}
// Rate limiting: delay between searches
await Task.Delay(DelayBetweenSearchesMs, cancellationToken);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}",
track.Title, track.PrimaryArtist);
}
}
if (matchedSongs.Count > 0)
{
// Cache matched tracks for 1 hour
await _cache.SetAsync(matchedTracksKey, matchedSongs, TimeSpan.FromHours(1));
_logger.LogInformation("✓ Cached {Matched}/{Total} matched tracks for {Playlist}",
matchedSongs.Count, missingTracks.Count, playlistName);
}
else
{
_logger.LogInformation("No tracks matched for {Playlist}", playlistName);
}
}
/// <summary>
/// Calculates artist match score ensuring ALL artists are present.
/// Penalizes if artist counts don't match or if any artist is missing.
/// </summary>
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
{
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
return 0;
// Build list of all song artists (main + contributors)
var allSongArtists = new List<string> { songMainArtist };
allSongArtists.AddRange(songContributors);
// If artist counts differ significantly, penalize
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
return 0;
// Check that each Spotify artist has a good match in song artists
var spotifyScores = new List<double>();
foreach (var spotifyArtist in spotifyArtists)
{
var bestMatch = allSongArtists.Max(songArtist =>
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
spotifyScores.Add(bestMatch);
}
// Check that each song artist has a good match in Spotify artists
var songScores = new List<double>();
foreach (var songArtist in allSongArtists)
{
var bestMatch = spotifyArtists.Max(spotifyArtist =>
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
songScores.Add(bestMatch);
}
// Average all scores - this ensures ALL artists must match well
var allScores = spotifyScores.Concat(songScores);
var avgScore = allScores.Average();
// Penalize if any individual artist match is poor (< 70)
var minScore = allScores.Min();
if (minScore < 70)
avgScore *= 0.7; // 30% penalty for poor individual match
return avgScore;
}
}

View File

@@ -28,6 +28,7 @@ public class SquidWTFDownloadService : BaseDownloadService
private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
protected override string ProviderName => "squidwtf";
@@ -48,23 +49,39 @@ public class SquidWTFDownloadService : BaseDownloadService
_apiUrls = apiUrls;
}
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action)
{
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try
{
var baseUrl = _apiUrls[_currentUrlIndex];
Logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
baseUrl, attempt + 1, _apiUrls.Count);
return await action(baseUrl);
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]);
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
if (attempt == _apiUrls.Count - 1)
{
Logger.LogError("All SquidWTF endpoints failed");
Logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
throw;
}
}

View File

@@ -23,6 +23,7 @@ public class SquidWTFMetadataService : IMusicMetadataService
private readonly RedisCacheService _cache;
private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
public SquidWTFMetadataService(
IHttpClientFactory httpClientFactory,
@@ -43,25 +44,52 @@ public class SquidWTFMetadataService : IMusicMetadataService
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0");
}
private string GetCurrentBaseUrl() => _apiUrls[_currentUrlIndex];
/// <summary>
/// Gets the next URL in round-robin fashion to distribute load across providers
/// </summary>
private string GetNextBaseUrl()
{
lock (_urlIndexLock)
{
var url = _apiUrls[_currentUrlIndex];
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
return url;
}
}
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
{
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try
{
var baseUrl = _apiUrls[_currentUrlIndex];
_logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
baseUrl, attempt + 1, _apiUrls.Count);
return await action(baseUrl);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]);
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
_logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
if (attempt == _apiUrls.Count - 1)
{
_logger.LogError("All SquidWTF endpoints failed");
_logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
return defaultValue;
}
}

View File

@@ -14,6 +14,7 @@ public class SquidWTFStartupValidator : BaseStartupValidator
private readonly SquidWTFSettings _settings;
private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
public override string ServiceName => "SquidWTF";
@@ -24,22 +25,37 @@ public class SquidWTFStartupValidator : BaseStartupValidator
_apiUrls = apiUrls;
}
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
{
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try
{
var baseUrl = _apiUrls[_currentUrlIndex];
return await action(baseUrl);
}
catch
{
WriteDetail($"Endpoint {_apiUrls[_currentUrlIndex]} failed, trying next...");
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
WriteDetail($"Endpoint {baseUrl} failed, trying next...");
if (attempt == _apiUrls.Count - 1)
{
WriteDetail($"All {_apiUrls.Count} endpoints failed");
return defaultValue;
}
}

View File

@@ -6,12 +6,14 @@ services:
# Redis is only accessible internally - no external port exposure
expose:
- "6379"
command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru
command: redis-server --maxmemory 1gb --maxmemory-policy allkeys-lru --save 60 1 --appendonly yes
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 3s
retries: 3
volumes:
- ${REDIS_DATA_PATH:-./redis-data}:/data
networks:
- allstarr-network