add round robin load balancing so providers dont hate me

This commit is contained in:
2026-02-02 12:57:04 -05:00
parent 2bc2816191
commit 9dd49a2f43
7 changed files with 190 additions and 21 deletions

View File

@@ -250,6 +250,10 @@ Choose your preferred provider via the `MUSIC_SERVICE` environment variable. Add
|---------|-------------| |---------|-------------|
| `SquidWTF:Quality` | Preferred audio quality: `FLAC`, `MP3_320`, `MP3_128`. If not specified, the highest available quality for your account will be used | | `SquidWTF:Quality` | Preferred audio quality: `FLAC`, `MP3_320`, `MP3_128`. If not specified, the highest available quality for your account will be used |
**Load Balancing & Reliability:**
SquidWTF uses a round-robin load balancing strategy across multiple backup API endpoints to distribute requests evenly and prevent overwhelming any single provider. Each request automatically rotates to the next endpoint in the pool, with automatic fallback to other endpoints if one fails. This ensures high availability and prevents rate limiting by distributing load across multiple providers.
### Deezer Settings ### Deezer Settings
| Setting | Description | | Setting | Description |

View File

@@ -2821,13 +2821,14 @@ public class JellyfinController : ControllerBase
if (results.Count > 0) if (results.Count > 0)
{ {
// Fuzzy match to find best result // Fuzzy match to find best result
// Check that ALL artists match (not just some)
var bestMatch = results var bestMatch = results
.Select(song => new .Select(song => new
{ {
Song = song, Song = song,
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title), TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist), // Calculate artist score by checking ALL artists match
TotalScore = 0.0 ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
}) })
.Select(x => new .Select(x => new
{ {
@@ -3320,5 +3321,53 @@ public class JellyfinController : ControllerBase
} }
#endregion #endregion
/// <summary>
/// Calculates artist match score ensuring ALL artists are present.
/// Penalizes if artist counts don't match or if any artist is missing.
/// </summary>
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
{
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
return 0;
// Build list of all song artists (main + contributors)
var allSongArtists = new List<string> { songMainArtist };
allSongArtists.AddRange(songContributors);
// If artist counts differ significantly, penalize
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
return 0;
// Check that each Spotify artist has a good match in song artists
var spotifyScores = new List<double>();
foreach (var spotifyArtist in spotifyArtists)
{
var bestMatch = allSongArtists.Max(songArtist =>
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
spotifyScores.Add(bestMatch);
}
// Check that each song artist has a good match in Spotify artists
var songScores = new List<double>();
foreach (var songArtist in allSongArtists)
{
var bestMatch = spotifyArtists.Max(spotifyArtist =>
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
songScores.Add(bestMatch);
}
// Average all scores - this ensures ALL artists must match well
var allScores = spotifyScores.Concat(songScores);
var avgScore = allScores.Average();
// Penalize if any individual artist match is poor (< 70)
var minScore = allScores.Min();
if (minScore < 70)
avgScore *= 0.7; // 30% penalty for poor individual match
return avgScore;
}
} }
// force rebuild Sun Jan 25 13:22:47 EST 2026 // force rebuild Sun Jan 25 13:22:47 EST 2026

View File

@@ -23,11 +23,16 @@ static List<string> DecodeSquidWtfUrls()
var encodedUrls = new[] var encodedUrls = new[]
{ {
"aHR0cHM6Ly90cml0b24uc3F1aWQud3Rm", // triton "aHR0cHM6Ly90cml0b24uc3F1aWQud3Rm", // triton
"aHR0cHM6Ly9tb25vY2hyb21lLWFwaS5zYW1pZHkuY29t", // samidy
"aHR0cHM6Ly90aWRhbC1hcGkuYmluaW11bS5vcmc=", // binimum
"aHR0cHM6Ly90aWRhbC5raW5vcGx1cy5vbmxpbmU=", // kinoplus
"aHR0cHM6Ly9oaWZpLXR3by5zcG90aXNhdmVyLm5ldA==", // spoti-2
"aHR0cHM6Ly9oaWZpLW9uZS5zcG90aXNhdmVyLm5ldA==", // spoti-1
"aHR0cHM6Ly93b2xmLnFxZGwuc2l0ZQ==", // wolf "aHR0cHM6Ly93b2xmLnFxZGwuc2l0ZQ==", // wolf
"aHR0cDovL2h1bmQucXFkbC5zaXRl", // hund "aHR0cDovL2h1bmQucXFkbC5zaXRl", // hund
"aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==", // maus "aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=", // katze
"aHR0cHM6Ly92b2dlbC5xcWRsLnNpdGU=", // vogel "aHR0cHM6Ly92b2dlbC5xcWRsLnNpdGU=", // vogel
"aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=" // katze "aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==" // maus
}; };
return encodedUrls return encodedUrls

View File

@@ -154,12 +154,14 @@ public class SpotifyTrackMatchingService : BackgroundService
if (results.Count > 0) if (results.Count > 0)
{ {
// Fuzzy match to find best result // Fuzzy match to find best result
// Check that ALL artists match (not just some)
var bestMatch = results var bestMatch = results
.Select(song => new .Select(song => new
{ {
Song = song, Song = song,
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title), TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist) // Calculate artist score by checking ALL artists match
ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
}) })
.Select(x => new .Select(x => new
{ {
@@ -206,4 +208,52 @@ public class SpotifyTrackMatchingService : BackgroundService
_logger.LogInformation("No tracks matched for {Playlist}", playlistName); _logger.LogInformation("No tracks matched for {Playlist}", playlistName);
} }
} }
/// <summary>
/// Calculates artist match score ensuring ALL artists are present.
/// Penalizes if artist counts don't match or if any artist is missing.
/// </summary>
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
{
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
return 0;
// Build list of all song artists (main + contributors)
var allSongArtists = new List<string> { songMainArtist };
allSongArtists.AddRange(songContributors);
// If artist counts differ significantly, penalize
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
return 0;
// Check that each Spotify artist has a good match in song artists
var spotifyScores = new List<double>();
foreach (var spotifyArtist in spotifyArtists)
{
var bestMatch = allSongArtists.Max(songArtist =>
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
spotifyScores.Add(bestMatch);
}
// Check that each song artist has a good match in Spotify artists
var songScores = new List<double>();
foreach (var songArtist in allSongArtists)
{
var bestMatch = spotifyArtists.Max(spotifyArtist =>
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
songScores.Add(bestMatch);
}
// Average all scores - this ensures ALL artists must match well
var allScores = spotifyScores.Concat(songScores);
var avgScore = allScores.Average();
// Penalize if any individual artist match is poor (< 70)
var minScore = allScores.Min();
if (minScore < 70)
avgScore *= 0.7; // 30% penalty for poor individual match
return avgScore;
}
} }

View File

@@ -28,6 +28,7 @@ public class SquidWTFDownloadService : BaseDownloadService
private readonly List<string> _apiUrls; private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0; private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
protected override string ProviderName => "squidwtf"; protected override string ProviderName => "squidwtf";
@@ -48,23 +49,39 @@ public class SquidWTFDownloadService : BaseDownloadService
_apiUrls = apiUrls; _apiUrls = apiUrls;
} }
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action) private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action)
{ {
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++) for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{ {
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try try
{ {
var baseUrl = _apiUrls[_currentUrlIndex]; Logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
baseUrl, attempt + 1, _apiUrls.Count);
return await action(baseUrl); return await action(baseUrl);
} }
catch (Exception ex) catch (Exception ex)
{ {
Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]); Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
if (attempt == _apiUrls.Count - 1) if (attempt == _apiUrls.Count - 1)
{ {
Logger.LogError("All SquidWTF endpoints failed"); Logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
throw; throw;
} }
} }

View File

@@ -23,6 +23,7 @@ public class SquidWTFMetadataService : IMusicMetadataService
private readonly RedisCacheService _cache; private readonly RedisCacheService _cache;
private readonly List<string> _apiUrls; private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0; private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
public SquidWTFMetadataService( public SquidWTFMetadataService(
IHttpClientFactory httpClientFactory, IHttpClientFactory httpClientFactory,
@@ -43,25 +44,52 @@ public class SquidWTFMetadataService : IMusicMetadataService
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0"); "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0");
} }
private string GetCurrentBaseUrl() => _apiUrls[_currentUrlIndex]; /// <summary>
/// Gets the next URL in round-robin fashion to distribute load across providers
/// </summary>
private string GetNextBaseUrl()
{
lock (_urlIndexLock)
{
var url = _apiUrls[_currentUrlIndex];
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
return url;
}
}
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue) private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
{ {
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++) for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{ {
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try try
{ {
var baseUrl = _apiUrls[_currentUrlIndex]; _logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
baseUrl, attempt + 1, _apiUrls.Count);
return await action(baseUrl); return await action(baseUrl);
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]); _logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
if (attempt == _apiUrls.Count - 1) if (attempt == _apiUrls.Count - 1)
{ {
_logger.LogError("All SquidWTF endpoints failed"); _logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
return defaultValue; return defaultValue;
} }
} }

View File

@@ -14,6 +14,7 @@ public class SquidWTFStartupValidator : BaseStartupValidator
private readonly SquidWTFSettings _settings; private readonly SquidWTFSettings _settings;
private readonly List<string> _apiUrls; private readonly List<string> _apiUrls;
private int _currentUrlIndex = 0; private int _currentUrlIndex = 0;
private readonly object _urlIndexLock = new object();
public override string ServiceName => "SquidWTF"; public override string ServiceName => "SquidWTF";
@@ -24,22 +25,37 @@ public class SquidWTFStartupValidator : BaseStartupValidator
_apiUrls = apiUrls; _apiUrls = apiUrls;
} }
/// <summary>
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
/// This distributes load evenly across all providers while maintaining reliability.
/// </summary>
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue) private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
{ {
// Start with the next URL in round-robin to distribute load
var startIndex = 0;
lock (_urlIndexLock)
{
startIndex = _currentUrlIndex;
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
}
// Try all URLs starting from the round-robin selected one
for (int attempt = 0; attempt < _apiUrls.Count; attempt++) for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
{ {
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
var baseUrl = _apiUrls[urlIndex];
try try
{ {
var baseUrl = _apiUrls[_currentUrlIndex];
return await action(baseUrl); return await action(baseUrl);
} }
catch catch
{ {
WriteDetail($"Endpoint {_apiUrls[_currentUrlIndex]} failed, trying next..."); WriteDetail($"Endpoint {baseUrl} failed, trying next...");
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
if (attempt == _apiUrls.Count - 1) if (attempt == _apiUrls.Count - 1)
{ {
WriteDetail($"All {_apiUrls.Count} endpoints failed");
return defaultValue; return defaultValue;
} }
} }