fix(streaming): race squid metadata and handle upstream errors

allstarr/Controllers/JellyfinController.Audio.cs

@@ -1,5 +1,6 @@
 using allstarr.Services.Common;
 using Microsoft.AspNetCore.Mvc;
+using System.Net;
 
 namespace allstarr.Controllers;
 
@@ -193,23 +194,75 @@ public partial class JellyfinController
         }
         catch (Exception ex)
         {
-            if (ex is HttpRequestException httpRequestException && httpRequestException.StatusCode.HasValue)
-            {
-                _logger.LogError("Failed to stream external song {Provider}:{ExternalId}: {StatusCode}: {ReasonPhrase}",
-                    provider,
-                    externalId,
-                    (int)httpRequestException.StatusCode.Value,
-                    httpRequestException.StatusCode.Value);
-                _logger.LogDebug(ex, "Detailed streaming failure for external song {Provider}:{ExternalId}", provider, externalId);
-            }
-            else
-            {
-                _logger.LogError(ex, "Failed to stream external song {Provider}:{ExternalId}", provider, externalId);
-            }
-            return StatusCode(500, new { error = "Streaming failed" });
+            return HandleExternalStreamFailure(provider, externalId, ex);
         }
     }
 
+    private IActionResult HandleExternalStreamFailure(string provider, string externalId, Exception ex)
+    {
+        if (HttpContext.RequestAborted.IsCancellationRequested && ex is OperationCanceledException)
+        {
+            _logger.LogInformation("Client aborted external stream request for {Provider}:{ExternalId}", provider, externalId);
+            return StatusCode(499);
+        }
+
+        var (statusCode, errorMessage) = MapExternalStreamException(ex);
+
+        if (ex is HttpRequestException httpRequestException && httpRequestException.StatusCode.HasValue)
+        {
+            _logger.LogError("Failed to stream external song {Provider}:{ExternalId}: responding {StatusCode}; upstream returned {UpstreamStatus}: {ReasonPhrase}",
+                provider,
+                externalId,
+                statusCode,
+                (int)httpRequestException.StatusCode.Value,
+                httpRequestException.StatusCode.Value);
+            _logger.LogDebug(ex, "Detailed streaming failure for external song {Provider}:{ExternalId}", provider, externalId);
+        }
+        else
+        {
+            _logger.LogError(ex, "Failed to stream external song {Provider}:{ExternalId}: responding {StatusCode}",
+                provider, externalId, statusCode);
+        }
+
+        return StatusCode(statusCode, new { error = errorMessage });
+    }
+
+    private static (int statusCode, string errorMessage) MapExternalStreamException(Exception ex)
+    {
+        if (ex is TimeoutException || ex is TaskCanceledException)
+        {
+            return (StatusCodes.Status504GatewayTimeout, "External provider timed out");
+        }
+
+        if (ex is HttpRequestException httpRequestException)
+        {
+            return httpRequestException.StatusCode switch
+            {
+                HttpStatusCode.NotFound => (StatusCodes.Status404NotFound, "External track not found"),
+                HttpStatusCode.TooManyRequests => (StatusCodes.Status503ServiceUnavailable, "External provider is rate limiting requests"),
+                HttpStatusCode.BadGateway or
+                HttpStatusCode.ServiceUnavailable or
+                HttpStatusCode.GatewayTimeout or
+                HttpStatusCode.InternalServerError => (StatusCodes.Status503ServiceUnavailable, "External provider is unavailable"),
+                _ => (StatusCodes.Status502BadGateway, "External provider request failed")
+            };
+        }
+
+        if (ex is InvalidOperationException invalidOperationException &&
+            invalidOperationException.Message.Contains("endpoints", StringComparison.OrdinalIgnoreCase))
+        {
+            return (StatusCodes.Status503ServiceUnavailable, "External provider has no healthy endpoints");
+        }
+
+        if (ex.Message.Contains("endpoints failed", StringComparison.OrdinalIgnoreCase) ||
+            ex.Message.Contains("No SquidWTF endpoints", StringComparison.OrdinalIgnoreCase))
+        {
+            return (StatusCodes.Status503ServiceUnavailable, "External provider has no healthy endpoints");
+        }
+
+        return (StatusCodes.Status502BadGateway, "External stream failed");
+    }
+
     /// <summary>
     /// Universal audio endpoint - handles transcoding, format negotiation, and adaptive streaming.
     /// This is the primary endpoint used by Jellyfin Web and most clients.

allstarr/Controllers/JellyfinController.cs

@@ -820,9 +820,14 @@ public partial class JellyfinController : ControllerBase
     {
         try
         {
-            var (itemResult, statusCode) = await _proxyService.GetJsonAsyncInternal($"Items/{itemId}");
+            var (itemResult, statusCode) = await _proxyService.GetJsonAsync($"Items/{itemId}", null, Request.Headers);
             if (itemResult == null || statusCode != 200)
             {
+                _logger.LogDebug(
+                    "Skipping Jellyfin {ImageType} image tag resolution for Spotify playlist {PlaylistId}: upstream returned {StatusCode}",
+                    imageType,
+                    itemId,
+                    statusCode);
                 return null;
             }
 

allstarr/Controllers/SubSonicController.cs

@@ -173,8 +173,35 @@ public class SubsonicController : ControllerBase
         }
         catch (Exception ex)
         {
+            if (HttpContext.RequestAborted.IsCancellationRequested && ex is OperationCanceledException)
+            {
+                _logger.LogInformation("Client aborted external Subsonic stream request for {Id}", id);
+                return StatusCode(499);
+            }
+
+            if (ex is HttpRequestException httpRequestException && httpRequestException.StatusCode.HasValue)
+            {
+                var statusCode = httpRequestException.StatusCode == System.Net.HttpStatusCode.NotFound ? 404 : 503;
+                _logger.LogError(ex, "Failed to stream external Subsonic item {Id}: responding {StatusCode}; upstream returned {UpstreamStatus}",
+                    id, statusCode, (int)httpRequestException.StatusCode.Value);
+                return StatusCode(statusCode, new { error = statusCode == 404 ? "External track not found" : "External provider unavailable" });
+            }
+
+            if (ex is TimeoutException || ex is TaskCanceledException)
+            {
+                _logger.LogError(ex, "Timed out streaming external Subsonic item {Id}", id);
+                return StatusCode(504, new { error = "External provider timed out" });
+            }
+
+            if (ex is InvalidOperationException invalidOperationException &&
+                invalidOperationException.Message.Contains("endpoints", StringComparison.OrdinalIgnoreCase))
+            {
+                _logger.LogError(ex, "No healthy endpoints available for external Subsonic item {Id}", id);
+                return StatusCode(503, new { error = "External provider has no healthy endpoints" });
+            }
+
             _logger.LogError(ex, "Failed to stream external Subsonic item {Id}", id);
-            return StatusCode(500, new { error = "Failed to stream" });
+            return StatusCode(502, new { error = "External stream failed" });
         }
     }
 

allstarr/Program.cs

@@ -954,7 +954,7 @@ app.UseMiddleware<BotProbeBlockMiddleware>();
 // Request logging middleware (when DEBUG_LOG_ALL_REQUESTS=true)
 app.UseMiddleware<RequestLoggingMiddleware>();
 
-app.UseExceptionHandler(_ => { }); // Global exception handler
+app.UseExceptionHandler(); // Use registered GlobalExceptionHandler
 
 // Enable response compression EARLY in the pipeline
 app.UseResponseCompression();

allstarr/Services/Common/RoundRobinFallbackHelper.cs

@@ -288,6 +288,9 @@ public class RoundRobinFallbackHelper
             return await action(endpointsToRace[0], cancellationToken);
         }
 
+        _logger.LogInformation("Racing {Count} {Service} endpoints: {Endpoints}",
+            endpointsToRace.Count, _serviceName, string.Join(", ", endpointsToRace));
+
         using var raceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
         var tasks = new List<Task<(T result, string endpoint, bool success)>>();
 
@@ -320,7 +323,7 @@ public class RoundRobinFallbackHelper
 
             if (success)
             {
-                _logger.LogDebug("🏆 {Service} race won by {Endpoint}, canceling others", _serviceName, endpoint);
+                _logger.LogInformation("{Service} race won by {Endpoint}", _serviceName, endpoint);
                 raceCts.Cancel(); // Cancel all other requests
                 return result;
             }
@@ -328,6 +331,8 @@ public class RoundRobinFallbackHelper
             tasks.Remove(completedTask);
         }
 
+        _logger.LogError("All raced {Service} endpoints failed: {Endpoints}",
+            _serviceName, string.Join(", ", endpointsToRace));
         throw new Exception($"All {topN} {_serviceName} endpoints failed in race");
     }
 

allstarr/Services/SquidWTF/SquidWTFMetadataService.cs

@@ -583,50 +583,78 @@ public class SquidWTFMetadataService : TrackParserBase, IMusicMetadataService
     {
         if (externalProvider != "squidwtf") return null;
 
-        return await _fallbackHelper.TryWithFallbackAsync(async (baseUrl) =>
+        var raceCount = Math.Min(3, _fallbackHelper.EndpointCount);
+
+        if (raceCount > 1)
         {
-            // Per hifi-api spec: GET /info/?id={trackId} returns track metadata
-            var url = $"{baseUrl}/info/?id={externalId}";
+            _logger.LogInformation(
+                "Racing top {EndpointCount} SquidWTF endpoints for track {TrackId} metadata resolution",
+                raceCount,
+                externalId);
 
-            var response = await _httpClient.GetAsync(url, cancellationToken);
-            if (!response.IsSuccessStatusCode)
+            try
             {
-                throw new HttpRequestException($"HTTP {response.StatusCode}");
+                return await _fallbackHelper.RaceTopEndpointsAsync(
+                    raceCount,
+                    (baseUrl, ct) => FetchSongAsync(baseUrl, externalId, ct),
+                    cancellationToken);
             }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(
+                    ex,
+                    "Raced SquidWTF metadata lookup failed for track {TrackId}; falling back to sequential failover",
+                    externalId);
+            }
+        }
 
-            var json = await response.Content.ReadAsStringAsync(cancellationToken);
-            var result = JsonDocument.Parse(json);
+        return await _fallbackHelper.TryWithFallbackAsync(
+            baseUrl => FetchSongAsync(baseUrl, externalId, cancellationToken),
+            (Song?)null);
+    }
 
-            // Per hifi-api spec: response is { "version": "2.0", "data": { track object } }
-			if (!result.RootElement.TryGetProperty("data", out var track))
-			{
-				throw new InvalidOperationException($"SquidWTF /info response for track {externalId} did not contain data");
-			}
+    private async Task<Song> FetchSongAsync(string baseUrl, string externalId, CancellationToken cancellationToken)
+    {
+        var url = $"{baseUrl}/info/?id={externalId}";
 
-			var song = ParseTidalTrackFull(track);
+        _logger.LogInformation(
+            "Requesting SquidWTF track metadata for track {TrackId} from {Endpoint}",
+            externalId,
+            baseUrl);
+        _logger.LogDebug("Fetching SquidWTF track metadata from: {Url}", url);
 
-			// Enrich with MusicBrainz genres if missing (SquidWTF/Tidal doesn't provide genres)
-			if (_genreEnrichment != null && string.IsNullOrEmpty(song.Genre))
-			{
-				// Fire-and-forget: don't block the response waiting for genre enrichment
-				_ = Task.Run(async () =>
-				{
-					try
-					{
-						await _genreEnrichment.EnrichSongGenreAsync(song);
-					}
-					catch (Exception ex)
-					{
-						_logger.LogError(ex, "Failed to enrich genre for {Title}", song.Title);
-					}
-				});
-			}
+        using var response = await _httpClient.GetAsync(url, cancellationToken);
+        if (!response.IsSuccessStatusCode)
+        {
+            throw new HttpRequestException($"HTTP {response.StatusCode}", null, response.StatusCode);
+        }
 
-			// NOTE: Spotify ID conversion happens during download (in SquidWTFDownloadService)
-			// This avoids redundant conversions and ensures it's done in parallel with the download
+        var json = await response.Content.ReadAsStringAsync(cancellationToken);
+        using var result = JsonDocument.Parse(json);
 
-			return song;
-        }, (Song?)null);
+        if (!result.RootElement.TryGetProperty("data", out var track))
+        {
+            throw new InvalidOperationException($"SquidWTF /info response for track {externalId} did not contain data");
+        }
+
+        var song = ParseTidalTrackFull(track);
+
+        if (_genreEnrichment != null && string.IsNullOrEmpty(song.Genre))
+        {
+            _ = Task.Run(async () =>
+            {
+                try
+                {
+                    await _genreEnrichment.EnrichSongGenreAsync(song);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Failed to enrich genre for {Title}", song.Title);
+                }
+            });
+        }
+
+        return song;
     }
 
     public async Task<List<Song>> GetTrackRecommendationsAsync(string externalId, int limit = 20, CancellationToken cancellationToken = default)