mirror of
https://github.com/SoPat712/allstarr.git
synced 2026-02-09 23:55:10 -05:00
Optimize track matching with parallel batch processing
- Process tracks in batches of 11 (matches SquidWTF provider count) - Each batch runs 11 parallel searches (one per provider) - Wait 150ms between batches (not between individual tracks) - This is ~11x faster: 50 tracks now takes ~1 second instead of ~7.5 seconds - Round-robin in SquidWTF ensures each parallel request hits a different provider - Maintains rate limiting while maximizing throughput
This commit is contained in:
@@ -24,6 +24,7 @@ public class SpotifyTrackMatchingService : BackgroundService
|
||||
private readonly ILogger<SpotifyTrackMatchingService> _logger;
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private const int DelayBetweenSearchesMs = 150; // 150ms = ~6.6 searches/second to avoid rate limiting
|
||||
private const int BatchSize = 11; // Number of parallel searches (matches SquidWTF provider count)
|
||||
|
||||
public SpotifyTrackMatchingService(
|
||||
IOptions<SpotifyImportSettings> spotifySettings,
|
||||
@@ -179,72 +180,104 @@ public class SpotifyTrackMatchingService : BackgroundService
|
||||
var fuzzyMatches = 0;
|
||||
var noMatch = 0;
|
||||
|
||||
foreach (var spotifyTrack in spotifyTracks.OrderBy(t => t.Position))
|
||||
// Process tracks in batches for parallel searching
|
||||
var orderedTracks = spotifyTracks.OrderBy(t => t.Position).ToList();
|
||||
for (int i = 0; i < orderedTracks.Count; i += BatchSize)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested) break;
|
||||
|
||||
try
|
||||
var batch = orderedTracks.Skip(i).Take(BatchSize).ToList();
|
||||
_logger.LogDebug("Processing batch {Start}-{End} of {Total}",
|
||||
i + 1, Math.Min(i + BatchSize, orderedTracks.Count), orderedTracks.Count);
|
||||
|
||||
// Process all tracks in this batch in parallel
|
||||
var batchTasks = batch.Select(async spotifyTrack =>
|
||||
{
|
||||
Song? matchedSong = null;
|
||||
var matchType = "none";
|
||||
|
||||
// Try ISRC match first if available and enabled
|
||||
if (_spotifyApiSettings.PreferIsrcMatching && !string.IsNullOrEmpty(spotifyTrack.Isrc))
|
||||
try
|
||||
{
|
||||
matchedSong = await TryMatchByIsrcAsync(spotifyTrack.Isrc, metadataService);
|
||||
if (matchedSong != null)
|
||||
Song? matchedSong = null;
|
||||
var matchType = "none";
|
||||
|
||||
// Try ISRC match first if available and enabled
|
||||
if (_spotifyApiSettings.PreferIsrcMatching && !string.IsNullOrEmpty(spotifyTrack.Isrc))
|
||||
{
|
||||
matchType = "isrc";
|
||||
isrcMatches++;
|
||||
matchedSong = await TryMatchByIsrcAsync(spotifyTrack.Isrc, metadataService);
|
||||
if (matchedSong != null)
|
||||
{
|
||||
matchType = "isrc";
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to fuzzy matching
|
||||
if (matchedSong == null)
|
||||
{
|
||||
matchedSong = await TryMatchByFuzzyAsync(
|
||||
spotifyTrack.Title,
|
||||
spotifyTrack.Artists,
|
||||
metadataService);
|
||||
|
||||
if (matchedSong != null)
|
||||
{
|
||||
matchType = "fuzzy";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to fuzzy matching
|
||||
if (matchedSong == null)
|
||||
{
|
||||
matchedSong = await TryMatchByFuzzyAsync(
|
||||
spotifyTrack.Title,
|
||||
spotifyTrack.Artists,
|
||||
metadataService);
|
||||
|
||||
if (matchedSong != null)
|
||||
{
|
||||
matchType = "fuzzy";
|
||||
fuzzyMatches++;
|
||||
var matched = new MatchedTrack
|
||||
{
|
||||
Position = spotifyTrack.Position,
|
||||
SpotifyId = spotifyTrack.SpotifyId,
|
||||
SpotifyTitle = spotifyTrack.Title,
|
||||
SpotifyArtist = spotifyTrack.PrimaryArtist,
|
||||
Isrc = spotifyTrack.Isrc,
|
||||
MatchType = matchType,
|
||||
MatchedSong = matchedSong
|
||||
};
|
||||
|
||||
_logger.LogDebug(" #{Position} {Title} - {Artist} → {MatchType} match: {MatchedTitle}",
|
||||
spotifyTrack.Position, spotifyTrack.Title, spotifyTrack.PrimaryArtist,
|
||||
matchType, matchedSong.Title);
|
||||
|
||||
return (matched, matchType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug(" #{Position} {Title} - {Artist} → no match",
|
||||
spotifyTrack.Position, spotifyTrack.Title, spotifyTrack.PrimaryArtist);
|
||||
return (null, "none");
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedSong != null)
|
||||
catch (Exception ex)
|
||||
{
|
||||
matchedTracks.Add(new MatchedTrack
|
||||
{
|
||||
Position = spotifyTrack.Position,
|
||||
SpotifyId = spotifyTrack.SpotifyId,
|
||||
SpotifyTitle = spotifyTrack.Title,
|
||||
SpotifyArtist = spotifyTrack.PrimaryArtist,
|
||||
Isrc = spotifyTrack.Isrc,
|
||||
MatchType = matchType,
|
||||
MatchedSong = matchedSong
|
||||
});
|
||||
|
||||
_logger.LogDebug(" #{Position} {Title} - {Artist} → {MatchType} match: {MatchedTitle}",
|
||||
spotifyTrack.Position, spotifyTrack.Title, spotifyTrack.PrimaryArtist,
|
||||
matchType, matchedSong.Title);
|
||||
_logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}",
|
||||
spotifyTrack.Title, spotifyTrack.PrimaryArtist);
|
||||
return (null, "none");
|
||||
}
|
||||
}).ToList();
|
||||
|
||||
// Wait for all tracks in this batch to complete
|
||||
var batchResults = await Task.WhenAll(batchTasks);
|
||||
|
||||
// Collect results
|
||||
foreach (var (matched, matchType) in batchResults)
|
||||
{
|
||||
if (matched != null)
|
||||
{
|
||||
matchedTracks.Add(matched);
|
||||
if (matchType == "isrc") isrcMatches++;
|
||||
else if (matchType == "fuzzy") fuzzyMatches++;
|
||||
}
|
||||
else
|
||||
{
|
||||
noMatch++;
|
||||
_logger.LogDebug(" #{Position} {Title} - {Artist} → no match",
|
||||
spotifyTrack.Position, spotifyTrack.Title, spotifyTrack.PrimaryArtist);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
await Task.Delay(DelayBetweenSearchesMs, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
// Rate limiting between batches (not between individual tracks)
|
||||
if (i + BatchSize < orderedTracks.Count)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to match track: {Title} - {Artist}",
|
||||
spotifyTrack.Title, spotifyTrack.PrimaryArtist);
|
||||
await Task.Delay(DelayBetweenSearchesMs, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user