feat(catalog): project native track aliases
CI / build-and-test (push) Canceled after 0s
CI / release-critical-tests (push) Canceled after 0s
CI / csharp-format (push) Canceled after 0s
CI / webui (push) Canceled after 0s
CI / release-manifest (push) Canceled after 0s
CI / apple-contracts (push) Canceled after 0s
CI / compose-contracts (push) Canceled after 0s

This commit is contained in:
joshpatra committed 2026-09-22 14:59:50 -04:00
1 parent 02f7532508
commit 70e3aaaf76
8 files changed
+260 -6

No files matched your search

@@ -106,6 +106,43 @@ public sealed class LibraryIndexServiceTests : IAsyncLifetime
_service.ListAsync(UnlinkedContext(), "music"));
}
[Fact]
public async Task CanonicalTrack_ProjectsOneBackendAliasAcrossAuthorizedUsers()
{
var canonicalId = Guid.CreateVersion7();
await using (var db = await _factory.CreateDbContextAsync())
{
db.CanonicalRecordings.Add(new CanonicalRecordingRecord
{
Id = canonicalId,
TenantId = _tenantId,
CreatedByUserId = _userA,
Title = "Song",
IsProvisional = true,
CreatedAt = _clock.UtcNow,
UpdatedAt = _clock.UtcNow
});
await db.SaveChangesAsync();
}
var input = Input() with
{
CanonicalRecordingId = canonicalId,
AcceptedDecisionVersion = 1
};
await _service.UpsertAsync(Context(_userA, "principal-a", "music"), input);
await _service.UpsertAsync(Context(_userB, "principal-b", "music"), input);
await using var verification = await _factory.CreateDbContextAsync();
Assert.Equal(2, await verification.LibraryTracks.CountAsync());
var alias = Assert.Single(await verification.CanonicalCatalogAliases.ToListAsync());
Assert.Equal(
CanonicalCatalogKeys.NativeTrackNamespace("jellyfin", "backend"),
alias.Namespace);
Assert.Equal("local-item", alias.ExternalId);
Assert.Equal(canonicalId, alias.CanonicalEntityId);
}
[Fact]
public async Task IndexRejectsSignedUrlsAndSecretLikeProviderIds()
{
@@ -22,6 +22,9 @@ public sealed class CanonicalAliasMigrationTests
var now = new DateTimeOffset(2026, 9, 22, 12, 0, 0, TimeSpan.Zero);
var tenantId = Guid.CreateVersion7();
var userId = Guid.CreateVersion7();
var secondUserId = Guid.CreateVersion7();
var backendIdentityId = Guid.CreateVersion7();
var secondBackendIdentityId = Guid.CreateVersion7();
var accountId = Guid.CreateVersion7();
var provisionalRecordingId = Guid.CreateVersion7();
var musicBrainzRecordingId = Guid.CreateVersion7();
@@ -48,6 +51,39 @@ public sealed class CanonicalAliasMigrationTests
CreatedAt = now,
UpdatedAt = now
},
new PlatformUserRecord
{
Id = secondUserId,
TenantId = tenantId,
DisplayName = "Second catalog user",
Status = PlatformUserStatus.Active,
CreatedAt = now,
UpdatedAt = now
},
new BackendIdentityRecord
{
Id = backendIdentityId,
TenantId = tenantId,
UserId = userId,
BackendType = "jellyfin",
BackendInstanceId = "home",
PrincipalId = "catalog-owner",
DisplayName = "Catalog owner",
CreatedAt = now,
LastSeenAt = now
},
new BackendIdentityRecord
{
Id = secondBackendIdentityId,
TenantId = tenantId,
UserId = secondUserId,
BackendType = "jellyfin",
BackendInstanceId = "home",
PrincipalId = "second-catalog-user",
DisplayName = "Second catalog user",
CreatedAt = now,
LastSeenAt = now
},
new ProviderAccountRecord
{
Id = accountId,
@@ -87,7 +123,9 @@ public sealed class CanonicalAliasMigrationTests
Identity(
musicBrainzRecordingId,
accountId,
ProviderIdentityScope.Account));
ProviderIdentityScope.Account),
LibraryTrack(userId, backendIdentityId, "owner.flac"),
LibraryTrack(secondUserId, secondBackendIdentityId, "second.flac"));
await seed.SaveChangesAsync();
}
@@ -106,7 +144,7 @@ public sealed class CanonicalAliasMigrationTests
var aliases = await verification.CanonicalCatalogAliases
.OrderBy(item => item.Namespace)
.ToListAsync();
Assert.Equal(4, aliases.Count);
Assert.Equal(5, aliases.Count);
Assert.Contains(aliases, alias =>
alias.Namespace == "isrc" &&
alias.ExternalId == isrc &&
@@ -129,6 +167,10 @@ public sealed class CanonicalAliasMigrationTests
alias.Namespace == accountNamespace &&
alias.ExternalId == externalId &&
alias.CanonicalEntityId == musicBrainzRecordingId);
Assert.Contains(aliases, alias =>
alias.Namespace == CanonicalCatalogKeys.NativeTrackNamespace("jellyfin", "home") &&
alias.ExternalId == "native-item" &&
alias.CanonicalEntityId == provisionalRecordingId);
ProviderTrackIdentityRecord Identity(
Guid canonicalRecordingId,
@@ -152,5 +194,28 @@ public sealed class CanonicalAliasMigrationTests
CreatedAt = now,
UpdatedAt = now
};
LibraryTrackRecord LibraryTrack(
Guid ownerUserId,
Guid backendIdentity,
string fileName) => new()
{
Id = Guid.CreateVersion7(),
TenantId = tenantId,
OwnerUserId = ownerUserId,
BackendIdentityId = backendIdentity,
CanonicalRecordingId = provisionalRecordingId,
LibraryScopeId = "music",
Protocol = "jellyfin",
BackendInstanceId = "home",
BackendItemId = "native-item",
FilePath = $"/media/{fileName}",
Title = "Provider-only recording",
Artist = "Fixture artist",
ProviderIdsJson = "{}",
IndexedAt = now,
SourceModifiedAt = now,
UpdatedAt = now
};
}
}
@@ -14,6 +14,6 @@ public sealed class MigrationModelSnapshotTests
using var context = new AllstarrDbContext(options);
Assert.False(context.Database.HasPendingModelChanges());
Assert.Equal("20260922000029_ProjectCanonicalAliases", context.Database.GetMigrations().Last());
Assert.Equal("20260922000030_ProjectNativeTrackAliases", context.Database.GetMigrations().Last());
}
}
@@ -76,6 +76,16 @@ public static class CanonicalCatalogKeys
providerAccountId?.ToString("D") ?? string.Empty);
return $"provider:{Hash(descriptor)}";
}
public static string NativeTrackNamespace(string protocol, string backendInstanceId)
{
protocol = ProviderContractValidation.Catalog(protocol, nameof(protocol));
backendInstanceId = ProviderContractValidation.RequiredText(
backendInstanceId,
nameof(backendInstanceId),
200);
return $"native:{Hash($"{protocol}\n{backendInstanceId}")}";
}
}
internal static class CanonicalCatalogIdentityProjection
@@ -148,6 +158,39 @@ internal static class CanonicalCatalogIdentityProjection
[],
cancellationToken);
}
public static Task ProjectLibraryTrackAsync(
AllstarrDbContext db,
ProviderActorContext actor,
LibraryTrackRecord track,
DateTimeOffset observedAt,
CancellationToken cancellationToken)
{
if (!track.CanonicalRecordingId.HasValue)
{
return Task.CompletedTask;
}
var aliasNamespace = CanonicalCatalogKeys.NativeTrackNamespace(
track.Protocol,
track.BackendInstanceId);
return CanonicalCatalogEvidenceStore.RecordInContextAsync(
db,
actor,
new CanonicalCatalogEntityReference(
CanonicalCatalogEntityKind.Recording,
track.CanonicalRecordingId.Value),
new CanonicalCatalogSourceStamp(
"native-library-identity",
$"{track.SourceModifiedAt.UtcTicks}:{track.AcceptedDecisionVersion}",
CanonicalCatalogKeys.Hash($"{aliasNamespace}\n{track.BackendItemId}"),
1,
observedAt,
null),
[new CanonicalCatalogAliasInput(aliasNamespace, track.BackendItemId)],
[],
cancellationToken);
}
}
public interface ICanonicalCatalogEvidenceStore
@@ -152,6 +152,13 @@ public sealed class LibraryIndexService : ILibraryIndexService
db.LibraryTracks.Add(record);
}
await CanonicalCatalogIdentityProjection.ProjectLibraryTrackAsync(
db,
executionContext.RequireActor(),
record,
now,
cancellationToken);
db.AuditEvents.Add(new AuditEventRecord
{
Id = Guid.CreateVersion7(),
@@ -1512,6 +1512,12 @@ public sealed class TrackMatchCommandService(
{
selected.CanonicalRecordingId = identity.CanonicalRecordingId;
selected.UpdatedAt = now;
await CanonicalCatalogIdentityProjection.ProjectLibraryTrackAsync(
db,
CatalogActor(snapshot),
selected,
now,
cancellationToken);
}
var state = Enum.Parse<TrackMatchState>(decision.State.ToString(), true);
@@ -2447,6 +2453,16 @@ public sealed class TrackMatchCommandService(
? snapshot.OwnerUserId
: null);
private static ProviderActorContext CatalogActor(
ExternalMetadataSnapshotRecord snapshot) => new(
snapshot.TenantId,
ProviderActorKind.User,
snapshot.OwnerUserId,
new ProviderBackendPrincipal(
snapshot.Protocol,
snapshot.BackendInstanceId,
snapshot.BackendPrincipalId));
private static string CleanReason(string? value, string fallback)
{
var reason = string.IsNullOrWhiteSpace(value) ? fallback : value.Trim();
@@ -0,0 +1,86 @@
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace allstarr.Core.Storage.Migrations;
[DbContext(typeof(AllstarrDbContext))]
[Migration("20260922000030_ProjectNativeTrackAliases")]
public sealed class ProjectNativeTrackAliases : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(
"""
WITH candidates AS (
SELECT
track."Id" AS "LibraryTrackId",
track."TenantId",
track."CanonicalRecordingId",
'native:' || encode(sha256(convert_to(
track."Protocol" || E'\n' || track."BackendInstanceId",
'UTF8')), 'hex') AS "Namespace",
track."BackendItemId" AS "ExternalId",
encode(sha256(convert_to(track."BackendItemId", 'UTF8')), 'hex') AS "ExternalIdHash",
track."IndexedAt",
track."UpdatedAt"
FROM "library_tracks" track
WHERE track."CanonicalRecordingId" IS NOT NULL
), consistent AS (
SELECT
candidate."TenantId",
candidate."Namespace",
candidate."ExternalIdHash",
min(candidate."ExternalId") AS "ExternalId",
min(candidate."CanonicalRecordingId"::text)::uuid AS "CanonicalRecordingId",
min(candidate."LibraryTrackId"::text)::uuid AS "LibraryTrackId",
min(candidate."IndexedAt") AS "CreatedAt",
max(candidate."UpdatedAt") AS "LastSeenAt"
FROM candidates candidate
GROUP BY
candidate."TenantId",
candidate."Namespace",
candidate."ExternalIdHash"
HAVING count(DISTINCT candidate."ExternalId") = 1
AND count(DISTINCT candidate."CanonicalRecordingId") = 1
)
INSERT INTO "canonical_catalog_aliases"
("Id", "TenantId", "EntityKind", "CanonicalEntityId", "Namespace",
"ExternalId", "ExternalIdHash", "CreatedAt", "LastSeenAt")
SELECT
substr(encode(sha256(convert_to(
'canonical-alias-v1' || E'\n' || 'native' || E'\n' ||
consistent."LibraryTrackId"::text,
'UTF8')), 'hex'), 1, 32)::uuid,
consistent."TenantId",
'Recording',
consistent."CanonicalRecordingId",
consistent."Namespace",
consistent."ExternalId",
consistent."ExternalIdHash",
consistent."CreatedAt",
consistent."LastSeenAt"
FROM consistent
WHERE NOT EXISTS (
SELECT 1
FROM "canonical_catalog_aliases" alias
WHERE alias."TenantId" = consistent."TenantId"
AND alias."Namespace" = consistent."Namespace"
AND alias."EntityKind" = 'Recording'
AND alias."ExternalIdHash" = consistent."ExternalIdHash");
""");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(
"""
DELETE FROM "canonical_catalog_aliases" alias
USING "library_tracks" track
WHERE alias."Id" = substr(encode(sha256(convert_to(
'canonical-alias-v1' || E'\n' || 'native' || E'\n' || track."Id"::text,
'UTF8')), 'hex'), 1, 32)::uuid;
""");
}
}
@@ -338,12 +338,12 @@ Checkpoint as of 2026-09-22; the alias projection changes are not yet deployed:
- **Catalog client:** The bounded `/ws/2` client supports source-scoped caches and artist, release-group, release, recording, ISRC, media, and release-track reads. Fixtures cover response hierarchy, cache isolation, size limits, rate limiting, cancellation, negative caching, and 401/403 handling. On 2026-09-15, eight operator-authorized BrainzMash requests passed with responses below 78 KB and latency from 656 ms to 11.129 seconds. A literal `Sunroof` search ranked acoustic/remix editions ahead of the base recording, confirming that Allstarr must rank normalized title, artist credit, duration, and version evidence itself.
- **Atomic ingestion:** `MusicBrainzCatalogIngestService` validates the full payload before a serializable transaction, preserves separate editions, replaces provisional fields with canonical facts, and writes provenance through the shared evidence owner. Repeated input preserves IDs and creates no duplicate facts. A disposable PostgreSQL run passed 54 catalog, client, environment, migration-snapshot, and storage tests, including rollback of malformed media.
- **Discovery and refresh:** `MusicBrainzCatalogRefreshQueue` creates seven-day idempotency generations scoped by tenant, user, source, and revision. Recording discovery accepts at most 50 distinct editions. Release refresh accepts one release hierarchy and at most 64 credited artists before atomic ingestion. Both jobs preserve upstream retry delays, separate permanent hierarchy failures from transient failures, and stay outside search and playback requests. The focused lane passes 62/62 tests; a separate PostgreSQL run passes all 21 selected identity, discovery, and ingestion tests.
- **Identity projection:** Creating a recording now projects exact ISRC and MusicBrainz aliases through the shared evidence owner. Both the identity service and matching commands project accepted provider identities transactionally with account- or catalog-scoped namespaces, so the same provider ID cannot leak or collide across account boundaries. Concurrent match writers treat an alias insert race as a retryable identity write. A forward migration backfills those aliases and marks recordings without an MBID provisional. The isolated PostgreSQL lane passes all 18 selected identity, migration, manual-selection, automatic-fallback, concurrency, and model-snapshot tests.
- **Remaining:** Project legacy library, source-snapshot, and protocol identities into the catalog; add the relationship and image request shapes needed by Stage 3; and reconcile provisional records that begin without an MBID.
- **Identity projection:** Creating a recording now projects exact ISRC and MusicBrainz aliases through the shared evidence owner. Both the identity service and matching commands project accepted provider identities transactionally with account- or catalog-scoped namespaces, so the same provider ID cannot leak or collide across account boundaries. Indexed Jellyfin and Subsonic/OpenSubsonic items use a protocol-and-backend-instance namespace; two users seeing the same native item converge only when their canonical assignments agree. Conflicting historical native assignments remain unaliased rather than being silently merged. Concurrent match writers treat an alias insert race as a retryable identity write. Forward migrations backfill provider, signal, and consistent native aliases and mark recordings without an MBID provisional. The isolated PostgreSQL lanes pass all 22 unique selected identity, migration, native-index, manual-selection, automatic-fallback, concurrency, and model-snapshot tests.
- **Remaining:** Project remaining legacy source-snapshot and protocol identities into the catalog; add the relationship and image request shapes needed by Stage 3; and reconcile provisional records that begin without an MBID.
| Stage 2 checkpoint measure | Stage start | Current | Interpretation |
| --- | ---: | ---: | --- |
| Production source in the established C#/WebUI/Python scope | 135,882 | 136,934 | +1,052 lines implement the catalog graph, evidence owner, bounded client, ingestion/refresh jobs, and exact identity projection; migrations and tests are excluded, and no consolidation saving is claimed yet |
| Production source in the established C#/WebUI/Python scope | 135,882 | 137,000 | +1,118 lines implement the catalog graph, evidence owner, bounded client, ingestion/refresh jobs, and exact identity projection; migrations and tests are excluded, and no consolidation saving is claimed yet |
| Accepted provider-identity alias writers | none | 1 shared evidence path | New links, repeat links, conflict checks, and the forward migration use the same namespace and hashing rules |
### Stage 3: Replace provider-shaped search with catalog search