feat(spotify): multi-provider mappings and Deezer-first hardening
CI / build-and-test (push) Has been cancelled

This commit is contained in:
joshpatra committed 2026-05-05 12:26:17 -04:00
1 parent e39d907462
commit ec1b2b2b7a
8 files changed
+273 -19

No files matched your search

@@ -496,7 +496,7 @@ public partial class JellyfinController
}
// Look for the track in cache folder first
var cacheBasePath = "/tmp/allstarr-cache";
var cacheBasePath = Path.Combine(_configuration["Library:DownloadPath"] ?? "./downloads", "cache");
var cacheArtistPath = Path.Combine(cacheBasePath, AdminHelperService.SanitizeFileName(song.Artist));
var cacheAlbumPath = Path.Combine(cacheArtistPath, AdminHelperService.SanitizeFileName(song.Album));
+33 -2
View File
@@ -9,6 +9,7 @@ using allstarr.Services.Admin;
using allstarr.Services;
using allstarr.Filters;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
namespace allstarr.Controllers;
@@ -27,6 +28,7 @@ public class PlaylistController : ControllerBase
private readonly HttpClient _jellyfinHttpClient;
private readonly AdminHelperService _helperService;
private readonly IServiceProvider _serviceProvider;
private readonly IConfiguration _configuration;
private const string CacheDirectory = "/app/cache/spotify";
public PlaylistController(
@@ -37,6 +39,7 @@ public class PlaylistController : ControllerBase
SpotifyMappingService mappingService,
RedisCacheService cache,
IHttpClientFactory httpClientFactory,
IConfiguration configuration,
AdminHelperService helperService,
IServiceProvider serviceProvider,
SpotifyTrackMatchingService? matchingService = null)
@@ -49,6 +52,7 @@ public class PlaylistController : ControllerBase
_mappingService = mappingService;
_cache = cache;
_jellyfinHttpClient = httpClientFactory.CreateClient();
_configuration = configuration;
_helperService = helperService;
_serviceProvider = serviceProvider;
}
@@ -753,7 +757,7 @@ public class PlaylistController : ControllerBase
if (string.IsNullOrWhiteSpace(externalProvider) &&
globalMappingExt?.TargetType == "external")
{
externalProvider = NormalizeExternalProviderForDisplay(globalMappingExt.ExternalProvider);
externalProvider = ResolvePreferredExternalProvider(globalMappingExt);
}
// Fallback 3: derive provider from external item ID prefix (ext-{provider}-...)
@@ -864,7 +868,7 @@ public class PlaylistController : ControllerBase
else if (globalMapping.TargetType == "external")
{
isLocal = false;
externalProvider = NormalizeExternalProviderForDisplay(globalMapping.ExternalProvider);
externalProvider = ResolvePreferredExternalProvider(globalMapping);
isManualMapping = true;
manualMappingType = "external";
manualMappingId = globalMapping.ExternalId;
@@ -1766,6 +1770,33 @@ public class PlaylistController : ControllerBase
return trimmed;
}
private string? ResolvePreferredExternalProvider(SpotifyTrackMapping mapping)
{
var preferredProvider = GetCurrentMusicServiceProvider();
if (mapping.TryGetExternalTarget(preferredProvider, out var provider, out _))
{
return NormalizeExternalProviderForDisplay(provider);
}
return NormalizeExternalProviderForDisplay(mapping.ExternalProvider);
}
private string? GetCurrentMusicServiceProvider()
{
var backendType = _configuration.GetValue<BackendType>("Backend:Type");
var musicService = backendType == BackendType.Jellyfin
? _configuration.GetValue<MusicService>("Jellyfin:MusicService")
: _configuration.GetValue<MusicService>("Subsonic:MusicService");
return musicService switch
{
MusicService.Deezer => "deezer",
MusicService.Qobuz => "qobuz",
MusicService.SquidWTF => "squidwtf",
_ => null
};
}
/// <summary>
/// Rebuild all playlists from scratch (clear cache, fetch fresh data, re-match).
/// This is a manual bulk action across all playlists - used by "Rebuild All Remote" button.
@@ -30,6 +30,13 @@ public class SpotifyTrackMapping
/// External provider track ID (if TargetType is "external")
/// </summary>
public string? ExternalId { get; set; }
/// <summary>
/// Multi-provider external mappings for this Spotify track.
/// Keeps additional external provider IDs so rematch/rebuild can add
/// provider-specific mappings without replacing existing ones.
/// </summary>
public List<ExternalTrackMapping> ExternalMappings { get; set; } = new();
/// <summary>
/// Track metadata for display purposes
@@ -79,6 +86,55 @@ public class SpotifyTrackMapping
return false;
}
/// <summary>
/// Resolves the best external mapping target, preferring the requested provider when available.
/// </summary>
public bool TryGetExternalTarget(string? preferredProvider, out string provider, out string externalId)
{
provider = string.Empty;
externalId = string.Empty;
if (!string.IsNullOrWhiteSpace(preferredProvider))
{
var preferred = ExternalMappings.FirstOrDefault(m =>
string.Equals(m.Provider, preferredProvider, StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrWhiteSpace(m.ExternalId));
if (preferred != null)
{
provider = preferred.Provider;
externalId = preferred.ExternalId;
return true;
}
}
var first = ExternalMappings.FirstOrDefault(m =>
!string.IsNullOrWhiteSpace(m.Provider) && !string.IsNullOrWhiteSpace(m.ExternalId));
if (first != null)
{
provider = first.Provider;
externalId = first.ExternalId;
return true;
}
if (!string.IsNullOrWhiteSpace(ExternalProvider) && !string.IsNullOrWhiteSpace(ExternalId))
{
provider = ExternalProvider;
externalId = ExternalId;
return true;
}
return false;
}
}
public class ExternalTrackMapping
{
public required string Provider { get; set; }
public required string ExternalId { get; set; }
public string Source { get; set; } = "auto";
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
public DateTime? UpdatedAt { get; set; }
}
/// <summary>
@@ -62,7 +62,8 @@ public static class StreamQualityHelper
return StreamQuality.Original;
}
return MapBitRateToQuality((int)(maxBitrate / 1000));
// MaxStreamingBitrate is reported in bits per second.
return MapBitRateToQuality((int)maxBitrate);
}
// Check for audioBitRate (lowercase variant used by some clients)
@@ -181,10 +181,10 @@ public class DeezerDownloadService : BaseDownloadService
_ => ".mp3"
};
// Write to transcoded cache directory: {downloads}/transcoded/Artist/Album/song.ext
// Write to transcoded cache directory: {DownloadPath}/transcoded/Artist/Album/song.ext
// These files are cleaned up by CacheCleanupService based on CACHE_TRANSCODE_MINUTES TTL
var artistForPath = song.AlbumArtist ?? song.Artist;
var basePath = Path.Combine("downloads", "transcoded");
var basePath = Path.Combine(DownloadPath, "transcoded");
var outputPath = PathHelper.BuildTrackPath(basePath, artistForPath, song.Album, song.Title, song.Track, extension, "deezer", trackId);
// Create directories if they don't exist
@@ -157,7 +157,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<Song?> GetSongAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return null;
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return null;
var url = $"{BaseUrl}/track/{externalId}";
var response = await _httpClient.GetAsync(url, cancellationToken);
@@ -249,7 +249,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<Album?> GetAlbumAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return null;
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return null;
var url = $"{BaseUrl}/album/{externalId}";
var response = await _httpClient.GetAsync(url, cancellationToken);
@@ -291,7 +291,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<Artist?> GetArtistAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return null;
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return null;
var url = $"{BaseUrl}/artist/{externalId}";
var response = await _httpClient.GetAsync(url, cancellationToken);
@@ -308,7 +308,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<List<Album>> GetArtistAlbumsAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return new List<Album>();
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return new List<Album>();
var url = $"{BaseUrl}/artist/{externalId}/albums";
var response = await _httpClient.GetAsync(url, cancellationToken);
@@ -332,7 +332,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<List<Song>> GetArtistTracksAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return new List<Song>();
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return new List<Song>();
var url = $"{BaseUrl}/artist/{externalId}/top?limit=50";
var response = await _httpClient.GetAsync(url, cancellationToken);
@@ -613,7 +613,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<ExternalPlaylist?> GetPlaylistAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return null;
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return null;
try
{
@@ -637,7 +637,7 @@ public class DeezerMetadataService : TrackParserBase, IMusicMetadataService
public async Task<List<Song>> GetPlaylistTracksAsync(string externalProvider, string externalId, CancellationToken cancellationToken = default)
{
if (externalProvider != "deezer") return new List<Song>();
if (!string.Equals(externalProvider, "deezer", StringComparison.OrdinalIgnoreCase)) return new List<Song>();
try
{
@@ -31,6 +31,7 @@ public class SpotifyMappingService
if (mapping != null)
{
EnsureExternalMappingsConsistency(mapping);
_logger.LogDebug("Found mapping for Spotify ID {SpotifyId}: {TargetType}", spotifyId, mapping.TargetType);
}
@@ -44,6 +45,8 @@ public class SpotifyMappingService
/// </summary>
public async Task<bool> SaveMappingAsync(SpotifyTrackMapping mapping)
{
EnsureExternalMappingsConsistency(mapping);
// Validate mapping
if (string.IsNullOrEmpty(mapping.SpotifyId))
{
@@ -58,9 +61,9 @@ public class SpotifyMappingService
}
if (mapping.TargetType == "external" &&
(string.IsNullOrEmpty(mapping.ExternalProvider) || string.IsNullOrEmpty(mapping.ExternalId)))
!mapping.TryGetExternalTarget(preferredProvider: null, out _, out _))
{
_logger.LogWarning("Cannot save external mapping: ExternalProvider and ExternalId are required");
_logger.LogWarning("Cannot save external mapping: at least one external provider/id is required");
return false;
}
@@ -69,6 +72,37 @@ public class SpotifyMappingService
// Check if mapping already exists
var existingMapping = await GetMappingAsync(mapping.SpotifyId);
// For external mappings, merge provider-specific mappings so rematch/rebuild
// can retain multiple sources (e.g. SquidWTF + Deezer) for the same Spotify ID.
if (mapping.TargetType == "external")
{
if (existingMapping != null && existingMapping.TargetType == "local")
{
var localMappingWithExternalAlternatives = existingMapping;
MergeExternalMappings(localMappingWithExternalAlternatives, mapping);
localMappingWithExternalAlternatives.UpdatedAt = DateTime.UtcNow;
var saveLocalWithAlternatives = await _cache.SetAsync(key, localMappingWithExternalAlternatives, expiry: null);
if (saveLocalWithAlternatives)
{
await AddToAllMappingsSetAsync(localMappingWithExternalAlternatives.SpotifyId);
await InvalidateAllPlaylistStatsCachesAsync();
_logger.LogInformation(
"Saved external fallback for local mapping: Spotify {SpotifyId} -> {Provider}:{ExternalId}",
mapping.SpotifyId,
mapping.ExternalProvider,
mapping.ExternalId);
}
return saveLocalWithAlternatives;
}
if (existingMapping != null && existingMapping.TargetType == "external")
{
MergeExternalMappings(mapping, existingMapping);
}
}
// RULE 1: Never overwrite manual mappings with auto mappings
if (existingMapping != null &&
existingMapping.Source == "manual" &&
@@ -84,6 +118,7 @@ public class SpotifyMappingService
mapping.TargetType == "local")
{
_logger.LogInformation("🎉 UPGRADING: External → Local for {SpotifyId}", mapping.SpotifyId);
MergeExternalMappings(mapping, existingMapping);
// Allow the upgrade to proceed
}
@@ -106,8 +141,14 @@ public class SpotifyMappingService
if (existingMapping != null)
{
mapping.CreatedAt = existingMapping.CreatedAt;
if (mapping.TargetType == "local" || mapping.TargetType == "external")
{
MergeExternalMappings(mapping, existingMapping);
}
}
EnsureExternalMappingsConsistency(mapping);
// Save mapping (permanent - no TTL)
var success = await _cache.SetAsync(key, mapping, expiry: null);
@@ -168,6 +209,16 @@ public class SpotifyMappingService
TargetType = "external",
ExternalProvider = externalProvider,
ExternalId = externalId,
ExternalMappings = new List<ExternalTrackMapping>
{
new()
{
Provider = externalProvider,
ExternalId = externalId,
Source = "auto",
CreatedAt = DateTime.UtcNow
}
},
Metadata = metadata,
Source = "auto",
CreatedAt = DateTime.UtcNow
@@ -194,6 +245,20 @@ public class SpotifyMappingService
LocalId = localId,
ExternalProvider = externalProvider,
ExternalId = externalId,
ExternalMappings = targetType == "external" &&
!string.IsNullOrWhiteSpace(externalProvider) &&
!string.IsNullOrWhiteSpace(externalId)
? new List<ExternalTrackMapping>
{
new()
{
Provider = externalProvider,
ExternalId = externalId,
Source = "manual",
CreatedAt = DateTime.UtcNow
}
}
: new List<ExternalTrackMapping>(),
Metadata = metadata,
Source = "manual",
CreatedAt = DateTime.UtcNow,
@@ -377,6 +442,87 @@ public class SpotifyMappingService
_logger.LogWarning(ex, "Failed to invalidate playlist stats caches");
}
}
private static void MergeExternalMappings(SpotifyTrackMapping target, SpotifyTrackMapping source)
{
if (source.TryGetExternalTarget(preferredProvider: null, out var sourceProvider, out var sourceExternalId))
{
UpsertExternalMapping(
target.ExternalMappings,
sourceProvider,
sourceExternalId,
source.Source);
}
foreach (var mapping in source.ExternalMappings)
{
if (string.IsNullOrWhiteSpace(mapping.Provider) || string.IsNullOrWhiteSpace(mapping.ExternalId))
{
continue;
}
UpsertExternalMapping(
target.ExternalMappings,
mapping.Provider,
mapping.ExternalId,
mapping.Source);
}
}
private static void UpsertExternalMapping(
List<ExternalTrackMapping> externalMappings,
string provider,
string externalId,
string source)
{
var normalizedProvider = provider.Trim().ToLowerInvariant();
var existing = externalMappings.FirstOrDefault(m =>
string.Equals(m.Provider, normalizedProvider, StringComparison.OrdinalIgnoreCase));
if (existing == null)
{
externalMappings.Add(new ExternalTrackMapping
{
Provider = normalizedProvider,
ExternalId = externalId,
Source = source,
CreatedAt = DateTime.UtcNow
});
return;
}
if (!string.Equals(existing.ExternalId, externalId, StringComparison.Ordinal))
{
existing.ExternalId = externalId;
existing.UpdatedAt = DateTime.UtcNow;
}
if (!string.IsNullOrWhiteSpace(source))
{
existing.Source = source;
}
}
private static void EnsureExternalMappingsConsistency(SpotifyTrackMapping mapping)
{
mapping.ExternalMappings ??= new List<ExternalTrackMapping>();
if (!string.IsNullOrWhiteSpace(mapping.ExternalProvider) && !string.IsNullOrWhiteSpace(mapping.ExternalId))
{
UpsertExternalMapping(
mapping.ExternalMappings,
mapping.ExternalProvider,
mapping.ExternalId,
mapping.Source);
}
if (mapping.ExternalMappings.Count > 0)
{
var first = mapping.ExternalMappings[0];
mapping.ExternalProvider = first.Provider;
mapping.ExternalId = first.ExternalId;
}
}
}
/// <summary>
@@ -4,6 +4,7 @@ using allstarr.Models.Spotify;
using allstarr.Services.Common;
using allstarr.Services.Jellyfin;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options;
using System.Text.Json;
using Cronos;
@@ -36,6 +37,7 @@ public class SpotifyTrackMatchingService : BackgroundService
private readonly SpotifyMappingValidationService _validationService;
private readonly ILogger<SpotifyTrackMatchingService> _logger;
private readonly IServiceProvider _serviceProvider;
private readonly IConfiguration _configuration;
private const int DelayBetweenSearchesMs = 150; // 150ms = ~6.6 searches/second to avoid rate limiting
private const int BatchSize = 11; // Number of parallel searches (matches SquidWTF provider count)
private static readonly TimeSpan ExternalProviderSearchTimeout = TimeSpan.FromSeconds(30);
@@ -51,6 +53,7 @@ public class SpotifyTrackMatchingService : BackgroundService
SpotifyMappingService mappingService,
SpotifyMappingValidationService validationService,
IServiceProvider serviceProvider,
IConfiguration configuration,
ILogger<SpotifyTrackMatchingService> logger)
{
_spotifySettings = spotifySettings.Value;
@@ -59,6 +62,7 @@ public class SpotifyTrackMatchingService : BackgroundService
_mappingService = mappingService;
_validationService = validationService;
_serviceProvider = serviceProvider;
_configuration = configuration;
_logger = logger;
}
@@ -803,13 +807,13 @@ public class SpotifyTrackMatchingService : BackgroundService
if (globalMapping != null && globalMapping.TargetType == "external")
{
Song? mappedSong = null;
var preferredProvider = GetCurrentMusicServiceProvider();
if (!string.IsNullOrEmpty(globalMapping.ExternalProvider) &&
!string.IsNullOrEmpty(globalMapping.ExternalId))
if (globalMapping.TryGetExternalTarget(preferredProvider, out var mappedProvider, out var mappedExternalId))
{
mappedSong = await metadataService.GetSongAsync(
globalMapping.ExternalProvider,
globalMapping.ExternalId,
mappedProvider,
mappedExternalId,
trackCancellationToken);
}
@@ -2036,4 +2040,20 @@ public class SpotifyTrackMatchingService : BackgroundService
return song;
}
private string? GetCurrentMusicServiceProvider()
{
var backendType = _configuration.GetValue<BackendType>("Backend:Type");
var musicService = backendType == BackendType.Jellyfin
? _configuration.GetValue<MusicService>("Jellyfin:MusicService")
: _configuration.GetValue<MusicService>("Subsonic:MusicService");
return musicService switch
{
MusicService.Deezer => "deezer",
MusicService.Qobuz => "qobuz",
MusicService.SquidWTF => "squidwtf",
_ => null
};
}
}