mirror of
https://github.com/SoPat712/allstarr.git
synced 2026-02-10 07:58:39 -05:00
add round robin load balancing so providers dont hate me
This commit is contained in:
@@ -250,6 +250,10 @@ Choose your preferred provider via the `MUSIC_SERVICE` environment variable. Add
|
|||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| `SquidWTF:Quality` | Preferred audio quality: `FLAC`, `MP3_320`, `MP3_128`. If not specified, the highest available quality for your account will be used |
|
| `SquidWTF:Quality` | Preferred audio quality: `FLAC`, `MP3_320`, `MP3_128`. If not specified, the highest available quality for your account will be used |
|
||||||
|
|
||||||
|
**Load Balancing & Reliability:**
|
||||||
|
|
||||||
|
SquidWTF uses a round-robin load balancing strategy across multiple backup API endpoints to distribute requests evenly and prevent overwhelming any single provider. Each request automatically rotates to the next endpoint in the pool, with automatic fallback to other endpoints if one fails. This ensures high availability and prevents rate limiting by distributing load across multiple providers.
|
||||||
|
|
||||||
### Deezer Settings
|
### Deezer Settings
|
||||||
|
|
||||||
| Setting | Description |
|
| Setting | Description |
|
||||||
|
|||||||
@@ -2821,13 +2821,14 @@ public class JellyfinController : ControllerBase
|
|||||||
if (results.Count > 0)
|
if (results.Count > 0)
|
||||||
{
|
{
|
||||||
// Fuzzy match to find best result
|
// Fuzzy match to find best result
|
||||||
|
// Check that ALL artists match (not just some)
|
||||||
var bestMatch = results
|
var bestMatch = results
|
||||||
.Select(song => new
|
.Select(song => new
|
||||||
{
|
{
|
||||||
Song = song,
|
Song = song,
|
||||||
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
|
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
|
||||||
ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist),
|
// Calculate artist score by checking ALL artists match
|
||||||
TotalScore = 0.0
|
ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
|
||||||
})
|
})
|
||||||
.Select(x => new
|
.Select(x => new
|
||||||
{
|
{
|
||||||
@@ -3320,5 +3321,53 @@ public class JellyfinController : ControllerBase
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Calculates artist match score ensuring ALL artists are present.
|
||||||
|
/// Penalizes if artist counts don't match or if any artist is missing.
|
||||||
|
/// </summary>
|
||||||
|
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
|
||||||
|
{
|
||||||
|
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// Build list of all song artists (main + contributors)
|
||||||
|
var allSongArtists = new List<string> { songMainArtist };
|
||||||
|
allSongArtists.AddRange(songContributors);
|
||||||
|
|
||||||
|
// If artist counts differ significantly, penalize
|
||||||
|
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
|
||||||
|
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// Check that each Spotify artist has a good match in song artists
|
||||||
|
var spotifyScores = new List<double>();
|
||||||
|
foreach (var spotifyArtist in spotifyArtists)
|
||||||
|
{
|
||||||
|
var bestMatch = allSongArtists.Max(songArtist =>
|
||||||
|
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
|
||||||
|
spotifyScores.Add(bestMatch);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that each song artist has a good match in Spotify artists
|
||||||
|
var songScores = new List<double>();
|
||||||
|
foreach (var songArtist in allSongArtists)
|
||||||
|
{
|
||||||
|
var bestMatch = spotifyArtists.Max(spotifyArtist =>
|
||||||
|
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
|
||||||
|
songScores.Add(bestMatch);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Average all scores - this ensures ALL artists must match well
|
||||||
|
var allScores = spotifyScores.Concat(songScores);
|
||||||
|
var avgScore = allScores.Average();
|
||||||
|
|
||||||
|
// Penalize if any individual artist match is poor (< 70)
|
||||||
|
var minScore = allScores.Min();
|
||||||
|
if (minScore < 70)
|
||||||
|
avgScore *= 0.7; // 30% penalty for poor individual match
|
||||||
|
|
||||||
|
return avgScore;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// force rebuild Sun Jan 25 13:22:47 EST 2026
|
// force rebuild Sun Jan 25 13:22:47 EST 2026
|
||||||
|
|||||||
@@ -23,11 +23,16 @@ static List<string> DecodeSquidWtfUrls()
|
|||||||
var encodedUrls = new[]
|
var encodedUrls = new[]
|
||||||
{
|
{
|
||||||
"aHR0cHM6Ly90cml0b24uc3F1aWQud3Rm", // triton
|
"aHR0cHM6Ly90cml0b24uc3F1aWQud3Rm", // triton
|
||||||
|
"aHR0cHM6Ly9tb25vY2hyb21lLWFwaS5zYW1pZHkuY29t", // samidy
|
||||||
|
"aHR0cHM6Ly90aWRhbC1hcGkuYmluaW11bS5vcmc=", // binimum
|
||||||
|
"aHR0cHM6Ly90aWRhbC5raW5vcGx1cy5vbmxpbmU=", // kinoplus
|
||||||
|
"aHR0cHM6Ly9oaWZpLXR3by5zcG90aXNhdmVyLm5ldA==", // spoti-2
|
||||||
|
"aHR0cHM6Ly9oaWZpLW9uZS5zcG90aXNhdmVyLm5ldA==", // spoti-1
|
||||||
"aHR0cHM6Ly93b2xmLnFxZGwuc2l0ZQ==", // wolf
|
"aHR0cHM6Ly93b2xmLnFxZGwuc2l0ZQ==", // wolf
|
||||||
"aHR0cDovL2h1bmQucXFkbC5zaXRl", // hund
|
"aHR0cDovL2h1bmQucXFkbC5zaXRl", // hund
|
||||||
"aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==", // maus
|
"aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=", // katze
|
||||||
"aHR0cHM6Ly92b2dlbC5xcWRsLnNpdGU=", // vogel
|
"aHR0cHM6Ly92b2dlbC5xcWRsLnNpdGU=", // vogel
|
||||||
"aHR0cHM6Ly9rYXR6ZS5xcWRsLnNpdGU=" // katze
|
"aHR0cHM6Ly9tYXVzLnFxZGwuc2l0ZQ==" // maus
|
||||||
};
|
};
|
||||||
|
|
||||||
return encodedUrls
|
return encodedUrls
|
||||||
|
|||||||
@@ -154,12 +154,14 @@ public class SpotifyTrackMatchingService : BackgroundService
|
|||||||
if (results.Count > 0)
|
if (results.Count > 0)
|
||||||
{
|
{
|
||||||
// Fuzzy match to find best result
|
// Fuzzy match to find best result
|
||||||
|
// Check that ALL artists match (not just some)
|
||||||
var bestMatch = results
|
var bestMatch = results
|
||||||
.Select(song => new
|
.Select(song => new
|
||||||
{
|
{
|
||||||
Song = song,
|
Song = song,
|
||||||
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
|
TitleScore = FuzzyMatcher.CalculateSimilarity(track.Title, song.Title),
|
||||||
ArtistScore = FuzzyMatcher.CalculateSimilarity(track.PrimaryArtist, song.Artist)
|
// Calculate artist score by checking ALL artists match
|
||||||
|
ArtistScore = CalculateArtistMatchScore(track.Artists, song.Artist, song.Contributors)
|
||||||
})
|
})
|
||||||
.Select(x => new
|
.Select(x => new
|
||||||
{
|
{
|
||||||
@@ -206,4 +208,52 @@ public class SpotifyTrackMatchingService : BackgroundService
|
|||||||
_logger.LogInformation("No tracks matched for {Playlist}", playlistName);
|
_logger.LogInformation("No tracks matched for {Playlist}", playlistName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Calculates artist match score ensuring ALL artists are present.
|
||||||
|
/// Penalizes if artist counts don't match or if any artist is missing.
|
||||||
|
/// </summary>
|
||||||
|
private static double CalculateArtistMatchScore(List<string> spotifyArtists, string songMainArtist, List<string> songContributors)
|
||||||
|
{
|
||||||
|
if (spotifyArtists.Count == 0 || string.IsNullOrEmpty(songMainArtist))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// Build list of all song artists (main + contributors)
|
||||||
|
var allSongArtists = new List<string> { songMainArtist };
|
||||||
|
allSongArtists.AddRange(songContributors);
|
||||||
|
|
||||||
|
// If artist counts differ significantly, penalize
|
||||||
|
var countDiff = Math.Abs(spotifyArtists.Count - allSongArtists.Count);
|
||||||
|
if (countDiff > 1) // Allow 1 artist difference (sometimes features are listed differently)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
// Check that each Spotify artist has a good match in song artists
|
||||||
|
var spotifyScores = new List<double>();
|
||||||
|
foreach (var spotifyArtist in spotifyArtists)
|
||||||
|
{
|
||||||
|
var bestMatch = allSongArtists.Max(songArtist =>
|
||||||
|
FuzzyMatcher.CalculateSimilarity(spotifyArtist, songArtist));
|
||||||
|
spotifyScores.Add(bestMatch);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that each song artist has a good match in Spotify artists
|
||||||
|
var songScores = new List<double>();
|
||||||
|
foreach (var songArtist in allSongArtists)
|
||||||
|
{
|
||||||
|
var bestMatch = spotifyArtists.Max(spotifyArtist =>
|
||||||
|
FuzzyMatcher.CalculateSimilarity(songArtist, spotifyArtist));
|
||||||
|
songScores.Add(bestMatch);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Average all scores - this ensures ALL artists must match well
|
||||||
|
var allScores = spotifyScores.Concat(songScores);
|
||||||
|
var avgScore = allScores.Average();
|
||||||
|
|
||||||
|
// Penalize if any individual artist match is poor (< 70)
|
||||||
|
var minScore = allScores.Min();
|
||||||
|
if (minScore < 70)
|
||||||
|
avgScore *= 0.7; // 30% penalty for poor individual match
|
||||||
|
|
||||||
|
return avgScore;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ public class SquidWTFDownloadService : BaseDownloadService
|
|||||||
|
|
||||||
private readonly List<string> _apiUrls;
|
private readonly List<string> _apiUrls;
|
||||||
private int _currentUrlIndex = 0;
|
private int _currentUrlIndex = 0;
|
||||||
|
private readonly object _urlIndexLock = new object();
|
||||||
|
|
||||||
protected override string ProviderName => "squidwtf";
|
protected override string ProviderName => "squidwtf";
|
||||||
|
|
||||||
@@ -48,23 +49,39 @@ public class SquidWTFDownloadService : BaseDownloadService
|
|||||||
_apiUrls = apiUrls;
|
_apiUrls = apiUrls;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
|
||||||
|
/// This distributes load evenly across all providers while maintaining reliability.
|
||||||
|
/// </summary>
|
||||||
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action)
|
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action)
|
||||||
{
|
{
|
||||||
|
// Start with the next URL in round-robin to distribute load
|
||||||
|
var startIndex = 0;
|
||||||
|
lock (_urlIndexLock)
|
||||||
|
{
|
||||||
|
startIndex = _currentUrlIndex;
|
||||||
|
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try all URLs starting from the round-robin selected one
|
||||||
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
||||||
{
|
{
|
||||||
|
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
|
||||||
|
var baseUrl = _apiUrls[urlIndex];
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var baseUrl = _apiUrls[_currentUrlIndex];
|
Logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
|
||||||
|
baseUrl, attempt + 1, _apiUrls.Count);
|
||||||
return await action(baseUrl);
|
return await action(baseUrl);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]);
|
Logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
|
||||||
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
|
||||||
|
|
||||||
if (attempt == _apiUrls.Count - 1)
|
if (attempt == _apiUrls.Count - 1)
|
||||||
{
|
{
|
||||||
Logger.LogError("All SquidWTF endpoints failed");
|
Logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ public class SquidWTFMetadataService : IMusicMetadataService
|
|||||||
private readonly RedisCacheService _cache;
|
private readonly RedisCacheService _cache;
|
||||||
private readonly List<string> _apiUrls;
|
private readonly List<string> _apiUrls;
|
||||||
private int _currentUrlIndex = 0;
|
private int _currentUrlIndex = 0;
|
||||||
|
private readonly object _urlIndexLock = new object();
|
||||||
|
|
||||||
public SquidWTFMetadataService(
|
public SquidWTFMetadataService(
|
||||||
IHttpClientFactory httpClientFactory,
|
IHttpClientFactory httpClientFactory,
|
||||||
@@ -43,25 +44,52 @@ public class SquidWTFMetadataService : IMusicMetadataService
|
|||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0");
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0");
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetCurrentBaseUrl() => _apiUrls[_currentUrlIndex];
|
/// <summary>
|
||||||
|
/// Gets the next URL in round-robin fashion to distribute load across providers
|
||||||
|
/// </summary>
|
||||||
|
private string GetNextBaseUrl()
|
||||||
|
{
|
||||||
|
lock (_urlIndexLock)
|
||||||
|
{
|
||||||
|
var url = _apiUrls[_currentUrlIndex];
|
||||||
|
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
|
||||||
|
/// This distributes load evenly across all providers while maintaining reliability.
|
||||||
|
/// </summary>
|
||||||
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
|
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
|
||||||
{
|
{
|
||||||
|
// Start with the next URL in round-robin to distribute load
|
||||||
|
var startIndex = 0;
|
||||||
|
lock (_urlIndexLock)
|
||||||
|
{
|
||||||
|
startIndex = _currentUrlIndex;
|
||||||
|
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try all URLs starting from the round-robin selected one
|
||||||
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
||||||
{
|
{
|
||||||
|
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
|
||||||
|
var baseUrl = _apiUrls[urlIndex];
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var baseUrl = _apiUrls[_currentUrlIndex];
|
_logger.LogDebug("Trying endpoint {Endpoint} (attempt {Attempt}/{Total})",
|
||||||
|
baseUrl, attempt + 1, _apiUrls.Count);
|
||||||
return await action(baseUrl);
|
return await action(baseUrl);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
_logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", _apiUrls[_currentUrlIndex]);
|
_logger.LogWarning(ex, "Request failed with endpoint {Endpoint}, trying next...", baseUrl);
|
||||||
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
|
||||||
|
|
||||||
if (attempt == _apiUrls.Count - 1)
|
if (attempt == _apiUrls.Count - 1)
|
||||||
{
|
{
|
||||||
_logger.LogError("All SquidWTF endpoints failed");
|
_logger.LogError("All {Count} SquidWTF endpoints failed", _apiUrls.Count);
|
||||||
return defaultValue;
|
return defaultValue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ public class SquidWTFStartupValidator : BaseStartupValidator
|
|||||||
private readonly SquidWTFSettings _settings;
|
private readonly SquidWTFSettings _settings;
|
||||||
private readonly List<string> _apiUrls;
|
private readonly List<string> _apiUrls;
|
||||||
private int _currentUrlIndex = 0;
|
private int _currentUrlIndex = 0;
|
||||||
|
private readonly object _urlIndexLock = new object();
|
||||||
|
|
||||||
public override string ServiceName => "SquidWTF";
|
public override string ServiceName => "SquidWTF";
|
||||||
|
|
||||||
@@ -24,22 +25,37 @@ public class SquidWTFStartupValidator : BaseStartupValidator
|
|||||||
_apiUrls = apiUrls;
|
_apiUrls = apiUrls;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries the request with the next provider in round-robin, then falls back to others on failure.
|
||||||
|
/// This distributes load evenly across all providers while maintaining reliability.
|
||||||
|
/// </summary>
|
||||||
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
|
private async Task<T> TryWithFallbackAsync<T>(Func<string, Task<T>> action, T defaultValue)
|
||||||
{
|
{
|
||||||
|
// Start with the next URL in round-robin to distribute load
|
||||||
|
var startIndex = 0;
|
||||||
|
lock (_urlIndexLock)
|
||||||
|
{
|
||||||
|
startIndex = _currentUrlIndex;
|
||||||
|
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try all URLs starting from the round-robin selected one
|
||||||
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
for (int attempt = 0; attempt < _apiUrls.Count; attempt++)
|
||||||
{
|
{
|
||||||
|
var urlIndex = (startIndex + attempt) % _apiUrls.Count;
|
||||||
|
var baseUrl = _apiUrls[urlIndex];
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var baseUrl = _apiUrls[_currentUrlIndex];
|
|
||||||
return await action(baseUrl);
|
return await action(baseUrl);
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
WriteDetail($"Endpoint {_apiUrls[_currentUrlIndex]} failed, trying next...");
|
WriteDetail($"Endpoint {baseUrl} failed, trying next...");
|
||||||
_currentUrlIndex = (_currentUrlIndex + 1) % _apiUrls.Count;
|
|
||||||
|
|
||||||
if (attempt == _apiUrls.Count - 1)
|
if (attempt == _apiUrls.Count - 1)
|
||||||
{
|
{
|
||||||
|
WriteDetail($"All {_apiUrls.Count} endpoints failed");
|
||||||
return defaultValue;
|
return defaultValue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user