mirror of
https://github.com/SoPat712/allstarr.git
synced 2026-04-27 03:53:10 -04:00
feat(search): implement fifo queue merge scoring
This commit is contained in:
@@ -57,7 +57,7 @@ public class JellyfinSearchInterleaveTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void InterleaveByScore_TiedRounds_AlternatesSourcesInsteadOfDrainingPrimary()
|
public void InterleaveByScore_TiedScores_PreferPrimaryQueueHead()
|
||||||
{
|
{
|
||||||
var controller = CreateController();
|
var controller = CreateController();
|
||||||
var primary = new List<Dictionary<string, object?>>
|
var primary = new List<Dictionary<string, object?>>
|
||||||
@@ -73,11 +73,11 @@ public class JellyfinSearchInterleaveTests
|
|||||||
|
|
||||||
var result = InvokeInterleaveByScore(controller, primary, secondary, "bts", 0.0);
|
var result = InvokeInterleaveByScore(controller, primary, secondary, "bts", 0.0);
|
||||||
|
|
||||||
Assert.Equal(["p1", "s1", "p2", "s2"], result.Select(GetId));
|
Assert.Equal(["p1", "p2", "s1", "s2"], result.Select(GetId));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void InterleaveByScore_StrongerLaterPrimaryHead_CanLeadSubsequentRoundWithoutReordering()
|
public void InterleaveByScore_StrongerLaterPrimaryHead_DoesNotBypassCurrentQueueHead()
|
||||||
{
|
{
|
||||||
var controller = CreateController();
|
var controller = CreateController();
|
||||||
var primary = new List<Dictionary<string, object?>>
|
var primary = new List<Dictionary<string, object?>>
|
||||||
@@ -93,7 +93,70 @@ public class JellyfinSearchInterleaveTests
|
|||||||
|
|
||||||
var result = InvokeInterleaveByScore(controller, primary, secondary, "bts", 0.0);
|
var result = InvokeInterleaveByScore(controller, primary, secondary, "bts", 0.0);
|
||||||
|
|
||||||
Assert.Equal(["s1", "p1", "p2", "s2"], result.Select(GetId));
|
Assert.Equal(["s1", "s2", "p1", "p2"], result.Select(GetId));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void InterleaveByScore_JellyfinBoost_CanWinCloseHeadToHead()
|
||||||
|
{
|
||||||
|
var controller = CreateController();
|
||||||
|
var primary = new List<Dictionary<string, object?>>
|
||||||
|
{
|
||||||
|
CreateItem("luther remastered", "p1")
|
||||||
|
};
|
||||||
|
var secondary = new List<Dictionary<string, object?>>
|
||||||
|
{
|
||||||
|
CreateItem("luther", "s1")
|
||||||
|
};
|
||||||
|
|
||||||
|
var result = InvokeInterleaveByScore(controller, primary, secondary, "luther", 5.0);
|
||||||
|
|
||||||
|
Assert.Equal(["p1", "s1"], result.Select(GetId));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CalculateItemRelevanceScore_SongUsesArtistContext()
|
||||||
|
{
|
||||||
|
var controller = CreateController();
|
||||||
|
var withArtist = CreateTypedItem("Audio", "cardigan", "song-with-artist");
|
||||||
|
withArtist["Artists"] = new[] { "Taylor Swift" };
|
||||||
|
|
||||||
|
var withoutArtist = CreateTypedItem("Audio", "cardigan", "song-without-artist");
|
||||||
|
|
||||||
|
var withArtistScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", withArtist);
|
||||||
|
var withoutArtistScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", withoutArtist);
|
||||||
|
|
||||||
|
Assert.True(withArtistScore > withoutArtistScore);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CalculateItemRelevanceScore_AlbumUsesArtistContext()
|
||||||
|
{
|
||||||
|
var controller = CreateController();
|
||||||
|
var withArtist = CreateTypedItem("MusicAlbum", "folklore", "album-with-artist");
|
||||||
|
withArtist["AlbumArtist"] = "Taylor Swift";
|
||||||
|
|
||||||
|
var withoutArtist = CreateTypedItem("MusicAlbum", "folklore", "album-without-artist");
|
||||||
|
|
||||||
|
var withArtistScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", withArtist);
|
||||||
|
var withoutArtistScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", withoutArtist);
|
||||||
|
|
||||||
|
Assert.True(withArtistScore > withoutArtistScore);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void CalculateItemRelevanceScore_ArtistIgnoresNonNameMetadata()
|
||||||
|
{
|
||||||
|
var controller = CreateController();
|
||||||
|
var plainArtist = CreateTypedItem("MusicArtist", "Taylor Swift", "artist-plain");
|
||||||
|
var noisyArtist = CreateTypedItem("MusicArtist", "Taylor Swift", "artist-noisy");
|
||||||
|
noisyArtist["AlbumArtist"] = "Completely Different";
|
||||||
|
noisyArtist["Artists"] = new[] { "Someone Else" };
|
||||||
|
|
||||||
|
var plainScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", plainArtist);
|
||||||
|
var noisyScore = InvokeCalculateItemRelevanceScore(controller, "taylor swift", noisyArtist);
|
||||||
|
|
||||||
|
Assert.Equal(plainScore, noisyScore);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static JellyfinController CreateController()
|
private static JellyfinController CreateController()
|
||||||
@@ -119,6 +182,20 @@ public class JellyfinSearchInterleaveTests
|
|||||||
[primary, secondary, query, primaryBoost])!;
|
[primary, secondary, query, primaryBoost])!;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static double InvokeCalculateItemRelevanceScore(
|
||||||
|
JellyfinController controller,
|
||||||
|
string query,
|
||||||
|
Dictionary<string, object?> item)
|
||||||
|
{
|
||||||
|
var method = typeof(JellyfinController).GetMethod(
|
||||||
|
"CalculateItemRelevanceScore",
|
||||||
|
BindingFlags.Instance | BindingFlags.NonPublic);
|
||||||
|
|
||||||
|
Assert.NotNull(method);
|
||||||
|
|
||||||
|
return (double)method!.Invoke(controller, [query, item])!;
|
||||||
|
}
|
||||||
|
|
||||||
private static Dictionary<string, object?> CreateItem(string name, string? id = null)
|
private static Dictionary<string, object?> CreateItem(string name, string? id = null)
|
||||||
{
|
{
|
||||||
return new Dictionary<string, object?>
|
return new Dictionary<string, object?>
|
||||||
@@ -128,6 +205,13 @@ public class JellyfinSearchInterleaveTests
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static Dictionary<string, object?> CreateTypedItem(string type, string name, string id)
|
||||||
|
{
|
||||||
|
var item = CreateItem(name, id);
|
||||||
|
item["Type"] = type;
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
private static string GetName(Dictionary<string, object?> item)
|
private static string GetName(Dictionary<string, object?> item)
|
||||||
{
|
{
|
||||||
return item["Name"]?.ToString() ?? string.Empty;
|
return item["Name"]?.ToString() ?? string.Empty;
|
||||||
|
|||||||
@@ -753,8 +753,8 @@ public partial class JellyfinController
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Interleaves two sources while preserving each source's original order.
|
/// Merges two source queues without reordering either queue.
|
||||||
/// Scores only decide which source leads each round; they do not re-rank either source.
|
/// At each step, compare only the current head from each source and dequeue the winner.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private List<Dictionary<string, object?>> InterleaveByScore(
|
private List<Dictionary<string, object?>> InterleaveByScore(
|
||||||
List<Dictionary<string, object?>> primaryItems,
|
List<Dictionary<string, object?>> primaryItems,
|
||||||
@@ -764,24 +764,20 @@ public partial class JellyfinController
|
|||||||
{
|
{
|
||||||
var primaryScored = primaryItems.Select(item =>
|
var primaryScored = primaryItems.Select(item =>
|
||||||
{
|
{
|
||||||
var baseScore = CalculateItemRelevanceScore(query, item);
|
|
||||||
return new
|
return new
|
||||||
{
|
{
|
||||||
Item = item,
|
Item = item,
|
||||||
BaseScore = baseScore,
|
Score = Math.Min(100.0, CalculateItemRelevanceScore(query, item) + primaryBoost)
|
||||||
Score = Math.Min(100.0, baseScore + primaryBoost)
|
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
.ToList();
|
.ToList();
|
||||||
|
|
||||||
var secondaryScored = secondaryItems.Select(item =>
|
var secondaryScored = secondaryItems.Select(item =>
|
||||||
{
|
{
|
||||||
var baseScore = CalculateItemRelevanceScore(query, item);
|
|
||||||
return new
|
return new
|
||||||
{
|
{
|
||||||
Item = item,
|
Item = item,
|
||||||
BaseScore = baseScore,
|
Score = CalculateItemRelevanceScore(query, item)
|
||||||
Score = baseScore
|
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
.ToList();
|
.ToList();
|
||||||
@@ -794,19 +790,13 @@ public partial class JellyfinController
|
|||||||
var primaryCandidate = primaryScored[primaryIdx];
|
var primaryCandidate = primaryScored[primaryIdx];
|
||||||
var secondaryCandidate = secondaryScored[secondaryIdx];
|
var secondaryCandidate = secondaryScored[secondaryIdx];
|
||||||
|
|
||||||
var primaryLeadsRound = primaryCandidate.Score > secondaryCandidate.Score ||
|
if (primaryCandidate.Score >= secondaryCandidate.Score)
|
||||||
(primaryCandidate.Score == secondaryCandidate.Score &&
|
|
||||||
primaryCandidate.BaseScore >= secondaryCandidate.BaseScore);
|
|
||||||
|
|
||||||
if (primaryLeadsRound)
|
|
||||||
{
|
{
|
||||||
result.Add(primaryScored[primaryIdx++].Item);
|
result.Add(primaryScored[primaryIdx++].Item);
|
||||||
result.Add(secondaryScored[secondaryIdx++].Item);
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
result.Add(secondaryScored[secondaryIdx++].Item);
|
result.Add(secondaryScored[secondaryIdx++].Item);
|
||||||
result.Add(primaryScored[primaryIdx++].Item);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -824,142 +814,17 @@ public partial class JellyfinController
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Calculates query relevance for a search item.
|
/// Calculates query relevance using the product's per-type rules.
|
||||||
/// Title is primary; metadata context is secondary and down-weighted.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private double CalculateItemRelevanceScore(string query, Dictionary<string, object?> item)
|
private double CalculateItemRelevanceScore(string query, Dictionary<string, object?> item)
|
||||||
{
|
{
|
||||||
var title = GetItemName(item);
|
return GetItemType(item) switch
|
||||||
if (string.IsNullOrWhiteSpace(title))
|
|
||||||
{
|
{
|
||||||
return 0;
|
"Audio" => CalculateSongRelevanceScore(query, item),
|
||||||
}
|
"MusicAlbum" => CalculateAlbumRelevanceScore(query, item),
|
||||||
|
"MusicArtist" => CalculateArtistRelevanceScore(query, item),
|
||||||
var titleScore = FuzzyMatcher.CalculateSimilarityAggressive(query, title);
|
_ => CalculateArtistRelevanceScore(query, item)
|
||||||
var searchText = BuildItemSearchText(item, title);
|
};
|
||||||
|
|
||||||
if (string.Equals(searchText, title, StringComparison.OrdinalIgnoreCase))
|
|
||||||
{
|
|
||||||
return titleScore;
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadataScore = FuzzyMatcher.CalculateSimilarityAggressive(query, searchText);
|
|
||||||
var weightedMetadataScore = metadataScore * 0.85;
|
|
||||||
|
|
||||||
var baseScore = Math.Max(titleScore, weightedMetadataScore);
|
|
||||||
return ApplyQueryCoverageAdjustment(query, title, searchText, baseScore);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static double ApplyQueryCoverageAdjustment(string query, string title, string searchText, double baseScore)
|
|
||||||
{
|
|
||||||
var queryTokens = TokenizeForCoverage(query);
|
|
||||||
if (queryTokens.Count < 2)
|
|
||||||
{
|
|
||||||
return baseScore;
|
|
||||||
}
|
|
||||||
|
|
||||||
var titleCoverage = CalculateTokenCoverage(queryTokens, title);
|
|
||||||
var searchCoverage = string.Equals(searchText, title, StringComparison.OrdinalIgnoreCase)
|
|
||||||
? titleCoverage
|
|
||||||
: CalculateTokenCoverage(queryTokens, searchText);
|
|
||||||
|
|
||||||
var coverage = Math.Max(titleCoverage, searchCoverage);
|
|
||||||
|
|
||||||
if (coverage >= 0.999)
|
|
||||||
{
|
|
||||||
return Math.Min(100.0, baseScore + 3.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (coverage >= 0.8)
|
|
||||||
{
|
|
||||||
return baseScore * 0.9;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (coverage >= 0.6)
|
|
||||||
{
|
|
||||||
return baseScore * 0.72;
|
|
||||||
}
|
|
||||||
|
|
||||||
return baseScore * 0.5;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static double CalculateTokenCoverage(IReadOnlyList<string> queryTokens, string target)
|
|
||||||
{
|
|
||||||
var targetTokens = TokenizeForCoverage(target);
|
|
||||||
if (queryTokens.Count == 0 || targetTokens.Count == 0)
|
|
||||||
{
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
var matched = 0;
|
|
||||||
foreach (var queryToken in queryTokens)
|
|
||||||
{
|
|
||||||
if (targetTokens.Any(targetToken => IsTokenMatch(queryToken, targetToken)))
|
|
||||||
{
|
|
||||||
matched++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (double)matched / queryTokens.Count;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsTokenMatch(string queryToken, string targetToken)
|
|
||||||
{
|
|
||||||
return queryToken.Equals(targetToken, StringComparison.Ordinal) ||
|
|
||||||
queryToken.StartsWith(targetToken, StringComparison.Ordinal) ||
|
|
||||||
targetToken.StartsWith(queryToken, StringComparison.Ordinal);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IReadOnlyList<string> TokenizeForCoverage(string text)
|
|
||||||
{
|
|
||||||
if (string.IsNullOrWhiteSpace(text))
|
|
||||||
{
|
|
||||||
return Array.Empty<string>();
|
|
||||||
}
|
|
||||||
|
|
||||||
var normalized = NormalizeForCoverage(text);
|
|
||||||
var allTokens = normalized
|
|
||||||
.Split(' ', StringSplitOptions.RemoveEmptyEntries)
|
|
||||||
.Distinct(StringComparer.Ordinal)
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (allTokens.Count == 0)
|
|
||||||
{
|
|
||||||
return Array.Empty<string>();
|
|
||||||
}
|
|
||||||
|
|
||||||
var significant = allTokens
|
|
||||||
.Where(token => token.Length >= 2 && !SearchStopWords.Contains(token))
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
return significant.Count > 0
|
|
||||||
? significant
|
|
||||||
: allTokens.Where(token => token.Length >= 2).ToList();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string NormalizeForCoverage(string text)
|
|
||||||
{
|
|
||||||
var normalized = RemoveDiacritics(text).ToLowerInvariant();
|
|
||||||
normalized = normalized.Replace('&', ' ');
|
|
||||||
normalized = System.Text.RegularExpressions.Regex.Replace(normalized, @"[^\w\s]", " ");
|
|
||||||
normalized = System.Text.RegularExpressions.Regex.Replace(normalized, @"\s+", " ").Trim();
|
|
||||||
return normalized;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static string RemoveDiacritics(string text)
|
|
||||||
{
|
|
||||||
var normalized = text.Normalize(NormalizationForm.FormD);
|
|
||||||
var chars = new List<char>(normalized.Length);
|
|
||||||
|
|
||||||
foreach (var c in normalized)
|
|
||||||
{
|
|
||||||
if (System.Globalization.CharUnicodeInfo.GetUnicodeCategory(c) != System.Globalization.UnicodeCategory.NonSpacingMark)
|
|
||||||
{
|
|
||||||
chars.Add(c);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return new string(chars.ToArray()).Normalize(NormalizationForm.FormC);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -970,52 +835,90 @@ public partial class JellyfinController
|
|||||||
return GetItemStringValue(item, "Name");
|
return GetItemStringValue(item, "Name");
|
||||||
}
|
}
|
||||||
|
|
||||||
private string BuildItemSearchText(Dictionary<string, object?> item, string title)
|
private double CalculateSongRelevanceScore(string query, Dictionary<string, object?> item)
|
||||||
{
|
{
|
||||||
var parts = new List<string>();
|
var title = GetItemName(item);
|
||||||
|
var artistText = GetSongArtistText(item);
|
||||||
AddDistinct(parts, title);
|
return CalculateBestFuzzyScore(query, title, CombineSearchFields(title, artistText));
|
||||||
AddDistinct(parts, GetItemStringValue(item, "SortName"));
|
|
||||||
AddDistinct(parts, GetItemStringValue(item, "AlbumArtist"));
|
|
||||||
AddDistinct(parts, GetItemStringValue(item, "Artist"));
|
|
||||||
AddDistinct(parts, GetItemStringValue(item, "Album"));
|
|
||||||
|
|
||||||
foreach (var artist in GetItemStringList(item, "Artists").Take(3))
|
|
||||||
{
|
|
||||||
AddDistinct(parts, artist);
|
|
||||||
}
|
|
||||||
|
|
||||||
return string.Join(" ", parts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static readonly HashSet<string> SearchStopWords = new(StringComparer.Ordinal)
|
private double CalculateAlbumRelevanceScore(string query, Dictionary<string, object?> item)
|
||||||
{
|
{
|
||||||
"a",
|
var albumName = GetItemName(item);
|
||||||
"an",
|
var artistText = GetAlbumArtistText(item);
|
||||||
"and",
|
return CalculateBestFuzzyScore(query, albumName, CombineSearchFields(albumName, artistText));
|
||||||
"at",
|
}
|
||||||
"for",
|
|
||||||
"in",
|
|
||||||
"of",
|
|
||||||
"on",
|
|
||||||
"the",
|
|
||||||
"to",
|
|
||||||
"with",
|
|
||||||
"feat",
|
|
||||||
"ft"
|
|
||||||
};
|
|
||||||
|
|
||||||
private static void AddDistinct(List<string> values, string? value)
|
private double CalculateArtistRelevanceScore(string query, Dictionary<string, object?> item)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(value))
|
var artistName = GetItemName(item);
|
||||||
|
if (string.IsNullOrWhiteSpace(artistName))
|
||||||
{
|
{
|
||||||
return;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!values.Contains(value, StringComparer.OrdinalIgnoreCase))
|
return FuzzyMatcher.CalculateSimilarityAggressive(query, artistName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private double CalculateBestFuzzyScore(string query, params string?[] candidates)
|
||||||
|
{
|
||||||
|
var best = 0;
|
||||||
|
|
||||||
|
foreach (var candidate in candidates)
|
||||||
{
|
{
|
||||||
values.Add(value);
|
if (string.IsNullOrWhiteSpace(candidate))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
best = Math.Max(best, FuzzyMatcher.CalculateSimilarityAggressive(query, candidate));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string CombineSearchFields(params string?[] fields)
|
||||||
|
{
|
||||||
|
return string.Join(" ", fields.Where(field => !string.IsNullOrWhiteSpace(field)));
|
||||||
|
}
|
||||||
|
|
||||||
|
private string GetItemType(Dictionary<string, object?> item)
|
||||||
|
{
|
||||||
|
return GetItemStringValue(item, "Type");
|
||||||
|
}
|
||||||
|
|
||||||
|
private string GetSongArtistText(Dictionary<string, object?> item)
|
||||||
|
{
|
||||||
|
var artists = GetItemStringList(item, "Artists").Take(3).ToList();
|
||||||
|
if (artists.Count > 0)
|
||||||
|
{
|
||||||
|
return string.Join(" ", artists);
|
||||||
|
}
|
||||||
|
|
||||||
|
var albumArtist = GetItemStringValue(item, "AlbumArtist");
|
||||||
|
if (!string.IsNullOrWhiteSpace(albumArtist))
|
||||||
|
{
|
||||||
|
return albumArtist;
|
||||||
|
}
|
||||||
|
|
||||||
|
return GetItemStringValue(item, "Artist");
|
||||||
|
}
|
||||||
|
|
||||||
|
private string GetAlbumArtistText(Dictionary<string, object?> item)
|
||||||
|
{
|
||||||
|
var albumArtist = GetItemStringValue(item, "AlbumArtist");
|
||||||
|
if (!string.IsNullOrWhiteSpace(albumArtist))
|
||||||
|
{
|
||||||
|
return albumArtist;
|
||||||
|
}
|
||||||
|
|
||||||
|
var artists = GetItemStringList(item, "Artists").Take(3).ToList();
|
||||||
|
if (artists.Count > 0)
|
||||||
|
{
|
||||||
|
return string.Join(" ", artists);
|
||||||
|
}
|
||||||
|
|
||||||
|
return GetItemStringValue(item, "Artist");
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetItemStringValue(Dictionary<string, object?> item, string key)
|
private string GetItemStringValue(Dictionary<string, object?> item, string key)
|
||||||
|
|||||||
Reference in New Issue
Block a user