Fix importing duplicate backdrops from nfo

This commit is contained in:
Cody Robibero committed 2026-10-17 20:07:51 -04:00
1 parent 08c2ba6e0b
commit 146982aa99
6 files changed
+521 -3

No files matched your search

@@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Entities;
using Jellyfin.Server.ServerSetupApp;
using MediaBrowser.Controller;
using MediaBrowser.Model.IO;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace Jellyfin.Server.Migrations.Routines;
/// <summary>
/// Deletes backdrop files in the internal metadata folder that are byte-identical to another backdrop of the
/// same item, and their rows. Older servers saved a backdrop URL from an NFO file as a new file on each
/// replace-images refresh, so one item can hold thousands of copies of one image.
/// </summary>
[JellyfinMigration("2026-10-11T12:00:00", nameof(RemoveDuplicateBackdrops))]
[JellyfinMigrationBackup(JellyfinDb = true)]
public class RemoveDuplicateBackdrops : IAsyncMigrationRoutine
{
private const int BatchSize = 500;
private readonly IStartupLogger<RemoveDuplicateBackdrops> _logger;
private readonly IDbContextFactory<JellyfinDbContext> _dbContextFactory;
private readonly IServerApplicationHost _appHost;
private readonly IServerApplicationPaths _appPaths;
private readonly IFileSystem _fileSystem;
/// <summary>
/// Initializes a new instance of the <see cref="RemoveDuplicateBackdrops"/> class.
/// </summary>
/// <param name="logger">The startup logger.</param>
/// <param name="dbContextFactory">The database context factory.</param>
/// <param name="appHost">The application host.</param>
/// <param name="appPaths">The server application paths.</param>
/// <param name="fileSystem">The file system.</param>
public RemoveDuplicateBackdrops(
IStartupLogger<RemoveDuplicateBackdrops> logger,
IDbContextFactory<JellyfinDbContext> dbContextFactory,
IServerApplicationHost appHost,
IServerApplicationPaths appPaths,
IFileSystem fileSystem)
{
_logger = logger;
_dbContextFactory = dbContextFactory;
_appHost = appHost;
_appPaths = appPaths;
_fileSystem = fileSystem;
}
/// <inheritdoc/>
public async Task PerformAsync(CancellationToken cancellationToken)
{
var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
await using (context.ConfigureAwait(false))
{
var itemIds = await context.BaseItemImageInfos
.Where(e => e.ImageType == ImageInfoImageType.Backdrop)
.GroupBy(e => e.ItemId)
.Where(g => g.Count() > 1)
.Select(g => g.Key)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
if (itemIds.Count == 0)
{
_logger.LogInformation("No items with more than one backdrop found.");
return;
}
_logger.LogInformation("Checking {Count} items with more than one backdrop for duplicates.", itemIds.Count);
var metadataPath = Path.GetFullPath(_appPaths.InternalMetadataPath);
var duplicates = new List<(Guid Id, string Path)>();
foreach (var batch in itemIds.Chunk(BatchSize))
{
var backdrops = await context.BaseItemImageInfos
.AsNoTracking()
.Where(e => batch.Contains(e.ItemId) && e.ImageType == ImageInfoImageType.Backdrop)
.Select(e => new { e.ItemId, e.Id, e.Path })
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
foreach (var item in backdrops.GroupBy(e => e.ItemId))
{
cancellationToken.ThrowIfCancellationRequested();
var itemDuplicates = await FindDuplicatesAsync(item.Select(e => (e.Id, e.Path)), metadataPath, cancellationToken).ConfigureAwait(false);
if (itemDuplicates.Count > 0)
{
_logger.LogInformation("Found {Count} duplicate backdrops for item {ItemId}.", itemDuplicates.Count, item.Key);
duplicates.AddRange(itemDuplicates);
}
}
}
if (duplicates.Count == 0)
{
_logger.LogInformation("No duplicate backdrops found.");
return;
}
// Rows go before files, so a failure leaves an unreferenced file and never a row without its file.
var transaction = await context.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
await using (transaction.ConfigureAwait(false))
{
foreach (var batch in duplicates.Chunk(BatchSize))
{
var ids = batch.Select(e => e.Id).ToArray();
await context.BaseItemImageInfos
.Where(e => ids.Contains(e.Id))
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
}
var deletedFiles = duplicates.Count(e => TryDeleteFile(e.Path));
_logger.LogInformation("Removed {Rows} duplicate backdrop rows and {Files} files.", duplicates.Count, deletedFiles);
}
}
private async Task<List<(Guid Id, string Path)>> FindDuplicatesAsync(IEnumerable<(Guid Id, string Path)> backdrops, string metadataPath, CancellationToken cancellationToken)
{
var duplicates = new List<(Guid Id, string Path)>();
var sameLength = backdrops
.Select(e => (e.Id, Path: _appHost.ExpandVirtualPath(e.Path)))
.Where(e => Path.IsPathFullyQualified(e.Path))
.Select(e => (e.Id, Path: Path.GetFullPath(e.Path)))
.Where(e => _fileSystem.ContainsSubPath(metadataPath, e.Path))
.Select(e => (Backdrop: e, File: new FileInfo(e.Path)))
.Where(e => e.File.Exists)
.OrderBy(e => e.Backdrop.Path, StringComparer.Ordinal)
.GroupBy(e => e.File.Length, e => e.Backdrop)
.Where(g => g.Count() > 1);
foreach (var group in sameLength)
{
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var backdrop in group)
{
var hash = await TryHashFileAsync(backdrop.Path, cancellationToken).ConfigureAwait(false);
if (hash is not null && !seen.Add(hash))
{
duplicates.Add(backdrop);
}
}
}
return duplicates;
}
private async Task<string?> TryHashFileAsync(string path, CancellationToken cancellationToken)
{
try
{
var stream = File.OpenRead(path);
await using (stream.ConfigureAwait(false))
{
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
return Convert.ToHexString(hash);
}
}
catch (IOException ex)
{
_logger.LogWarning(ex, "Failed to read backdrop {Path}", path);
}
catch (UnauthorizedAccessException ex)
{
_logger.LogWarning(ex, "Permission denied reading backdrop {Path}", path);
}
return null;
}
private bool TryDeleteFile(string path)
{
try
{
File.Delete(path);
return true;
}
catch (IOException ex)
{
_logger.LogWarning(ex, "Failed to delete duplicate backdrop {Path}", path);
}
catch (UnauthorizedAccessException ex)
{
_logger.LogWarning(ex, "Permission denied deleting duplicate backdrop {Path}", path);
}
return false;
}
}
@@ -574,6 +574,12 @@ namespace MediaBrowser.Providers.Manager
var filenames = images.Select(i => Path.GetFileNameWithoutExtension(i.Path)).ToList();
// The image at the index is the one being replaced, so its name stays free to reuse.
if (index is not null && index.Value < filenames.Count)
{
filenames.RemoveAt(index.Value);
}
var current = 1;
while (filenames.Contains(numberedIndexPrefix + current.ToString(CultureInfo.InvariantCulture), StringComparison.OrdinalIgnoreCase))
{
@@ -825,7 +825,11 @@ namespace MediaBrowser.Providers.Manager
continue;
}
await ProviderManager.SaveImage(item, remoteImage.Url, remoteImage.Type, null, cancellationToken).ConfigureAwait(false);
int? imageIndex = item.AllowsMultipleImages(remoteImage.Type)
? foundImageTypes.Count(x => x == remoteImage.Type)
: null;
await ProviderManager.SaveImage(item, remoteImage.Url, remoteImage.Type, imageIndex, cancellationToken).ConfigureAwait(false);
refreshResult.UpdateType |= ItemUpdateType.ImageUpdate;
// remember imagetype that has just been downloaded
@@ -0,0 +1,60 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.Library;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Entities;
using MediaBrowser.Model.IO;
using MediaBrowser.Providers.Manager;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace Jellyfin.Providers.Tests.Manager
{
public sealed class ImageSaverTests : IDisposable
{
private readonly string _dir;
public ImageSaverTests()
{
_dir = Directory.CreateTempSubdirectory("jellyfin-imagesaver").FullName;
}
public void Dispose() => Directory.Delete(_dir, true);
[Fact]
public async Task SaveImage_ReplaceNumberedBackdrop_KeepsFileName()
{
var config = new Mock<IServerConfigurationManager>();
config.Setup(c => c.Configuration).Returns(new ServerConfiguration());
config.Setup(c => c.ApplicationPaths.InternalMetadataPath).Returns(Path.Combine(_dir, "metadata"));
BaseItem.ConfigurationManager = config.Object;
BaseItem.MediaSourceManager = Mock.Of<IMediaSourceManager>();
BaseItem.FileSystem = Mock.Of<IFileSystem>();
var item = new LocalItem { Id = Guid.NewGuid(), Path = Path.Combine(_dir, "movie.mkv") };
item.AddImage(new ItemImageInfo { Path = Path.Combine(_dir, "backdrop.jpg"), Type = ImageType.Backdrop });
item.AddImage(new ItemImageInfo { Path = Path.Combine(_dir, "backdrop1.jpg"), Type = ImageType.Backdrop });
var fileSystem = new Mock<IFileSystem>();
fileSystem.Setup(f => f.GetFileInfo(It.IsAny<string>())).Returns((string path) => new FileSystemMetadata { FullName = path });
var saver = new ImageSaver(config.Object, Mock.Of<ILibraryMonitor>(), fileSystem.Object, NullLogger.Instance);
await saver.SaveImage(item, new MemoryStream([1]), "image/jpeg", ImageType.Backdrop, 1, TestContext.Current.CancellationToken);
Assert.Equal(Path.Combine(_dir, "backdrop1.jpg"), item.GetImages(ImageType.Backdrop).ElementAt(1).Path);
fileSystem.Verify(f => f.DeleteFile(It.IsAny<string>()), Times.Never);
}
private sealed class LocalItem : BaseItem
{
public override bool SupportsLocalMetadata => true;
public override bool IsSaveLocalMetadataEnabled() => true;
}
}
}
@@ -191,6 +191,42 @@ namespace Jellyfin.Providers.Tests.Manager
Assert.Equal("12345", tmdbIdSeenBySecondProvider);
}
[Fact]
public async Task RefreshWithProviders_ReplaceImages_NfoBackdropReplacesExistingSlot()
{
var item = new Movie { Name = "Test Movie" };
item.AddImage(new ItemImageInfo { Path = "https://example.com/old.jpg", Type = ImageType.Backdrop });
var local = new Mock<ILocalMetadataProvider<Movie>>(MockBehavior.Loose);
local.Setup(p => p.Name).Returns("Local");
local.Setup(p => p.GetMetadata(It.IsAny<ItemInfo>(), It.IsAny<IDirectoryService>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(() =>
{
var result = new MetadataResult<Movie> { HasMetadata = true, Item = new Movie { Name = "Test Movie" } };
result.RemoteImages.Add(("https://example.com/fanart1.jpg", ImageType.Backdrop));
result.RemoteImages.Add(("https://example.com/poster.jpg", ImageType.Primary));
result.RemoteImages.Add(("https://example.com/fanart2.jpg", ImageType.Backdrop));
return result;
});
var providerManager = new Mock<IProviderManager>(MockBehavior.Loose);
var service = new TestMetadataService(providerManager.Object);
await service.RefreshWithProvidersInternal(
new MetadataResult<Movie> { Item = item },
new MovieInfo { Name = item.Name },
new MetadataRefreshOptions(Mock.Of<IDirectoryService>())
{
MetadataRefreshMode = MetadataRefreshMode.FullRefresh,
ImageRefreshMode = MetadataRefreshMode.FullRefresh,
ReplaceAllImages = true
},
[local.Object]).ConfigureAwait(true);
providerManager.Verify(p => p.SaveImage(item, "https://example.com/fanart1.jpg", ImageType.Backdrop, 0, It.IsAny<CancellationToken>()), Times.Once);
providerManager.Verify(p => p.SaveImage(item, "https://example.com/fanart2.jpg", ImageType.Backdrop, 1, It.IsAny<CancellationToken>()), Times.Once);
providerManager.Verify(p => p.SaveImage(item, "https://example.com/poster.jpg", ImageType.Primary, null, It.IsAny<CancellationToken>()), Times.Once);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
@@ -469,11 +505,11 @@ namespace Jellyfin.Providers.Tests.Manager
private sealed class TestMetadataService : MetadataService<Movie, MovieInfo>
{
public TestMetadataService()
public TestMetadataService(IProviderManager? providerManager = null)
: base(
Mock.Of<IServerConfigurationManager>(),
NullLogger<MetadataService<Movie, MovieInfo>>.Instance,
Mock.Of<IProviderManager>(),
providerManager ?? Mock.Of<IProviderManager>(),
Mock.Of<IFileSystem>(),
Mock.Of<ILibraryManager>(),
Mock.Of<IExternalDataManager>(),
@@ -0,0 +1,211 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Emby.Server.Implementations.IO;
using Jellyfin.Database.Implementations;
using Jellyfin.Database.Implementations.Entities;
using Jellyfin.Database.Implementations.Locking;
using Jellyfin.Database.Providers.Sqlite;
using Jellyfin.Server.Migrations.Routines;
using Jellyfin.Server.ServerSetupApp;
using MediaBrowser.Common.Configuration;
using MediaBrowser.Controller;
using MediaBrowser.Controller.Entities.Movies;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace Jellyfin.Server.Tests.Migrations;
public sealed class RemoveDuplicateBackdropsTests : IDisposable
{
private static readonly byte[] _image = [1, 2, 3, 4];
private static readonly byte[] _sameLengthImage = [4, 3, 2, 1];
private static readonly byte[] _longerImage = [1, 2, 3, 4, 5];
private readonly SqliteConnection _connection;
private readonly DbContextOptions<JellyfinDbContext> _dbOptions;
private readonly string _dir;
public RemoveDuplicateBackdropsTests()
{
_connection = new SqliteConnection("Data Source=:memory:");
_connection.Open();
_dbOptions = new DbContextOptionsBuilder<JellyfinDbContext>()
.UseSqlite(_connection)
.Options;
using var context = CreateDbContext();
context.Database.EnsureCreated();
_dir = Directory.CreateTempSubdirectory("jellyfin-backdrops").FullName;
Directory.CreateDirectory(Path.Combine(_dir, "metadata", "movie"));
Directory.CreateDirectory(Path.Combine(_dir, "metadata", "other"));
Directory.CreateDirectory(Path.Combine(_dir, "media", "movie"));
}
public void Dispose()
{
_connection.Dispose();
Directory.Delete(_dir, true);
}
[Fact]
public async Task Perform_IdenticalBackdrops_KeepsOneCopyPerItem()
{
var movie = new BaseItemEntity { Id = Guid.NewGuid(), Type = typeof(Movie).FullName! };
var other = new BaseItemEntity { Id = Guid.NewGuid(), Type = typeof(Movie).FullName! };
using (var ctx = CreateDbContext())
{
ctx.BaseItems.AddRange(movie, other);
ctx.BaseItemImageInfos.AddRange(
Image(movie, WriteImage("metadata/movie/backdrop.jpg", _image)),
Image(movie, WriteImage("metadata/movie/backdrop1.jpg", _image)),
Image(movie, WriteImage("metadata/movie/backdrop2.jpg", _image)),
Image(movie, WriteImage("metadata/movie/backdrop3.jpg", _sameLengthImage)),
Image(movie, WriteImage("metadata/movie/backdrop4.jpg", _longerImage)),
Image(movie, Path.Combine(_dir, "metadata/movie/missing.jpg")),
Image(movie, "https://example.com/backdrop.jpg"),
Image(movie, WriteImage("metadata/movie/poster.jpg", _image), ImageInfoImageType.Primary),
Image(other, WriteImage("metadata/other/backdrop.jpg", _image)));
await ctx.SaveChangesAsync(TestContext.Current.CancellationToken);
}
await CreateMigration().PerformAsync(TestContext.Current.CancellationToken);
Assert.False(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop1.jpg")));
Assert.False(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop2.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop3.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop4.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "metadata/movie/poster.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "metadata/other/backdrop.jpg")));
using (var ctx = CreateDbContext())
{
var remaining = ctx.BaseItemImageInfos
.Select(e => e.Path)
.AsEnumerable()
.Select(p => p.Replace(_dir, string.Empty, StringComparison.Ordinal).Replace('\\', '/'))
.Order(StringComparer.Ordinal)
.ToArray();
Assert.Equal(
[
"/metadata/movie/backdrop.jpg",
"/metadata/movie/backdrop3.jpg",
"/metadata/movie/backdrop4.jpg",
"/metadata/movie/missing.jpg",
"/metadata/movie/poster.jpg",
"/metadata/other/backdrop.jpg",
"https://example.com/backdrop.jpg"
],
remaining);
}
}
[Fact]
public async Task Perform_VirtualMetadataPath_RemovesCopies()
{
var movie = new BaseItemEntity { Id = Guid.NewGuid(), Type = typeof(Movie).FullName! };
WriteImage("metadata/movie/backdrop.jpg", _image);
WriteImage("metadata/movie/backdrop1.jpg", _image);
using (var ctx = CreateDbContext())
{
ctx.BaseItems.Add(movie);
ctx.BaseItemImageInfos.AddRange(
Image(movie, "%MetadataPath%/movie/backdrop.jpg"),
Image(movie, "%MetadataPath%/movie/backdrop1.jpg"));
await ctx.SaveChangesAsync(TestContext.Current.CancellationToken);
}
await CreateMigration().PerformAsync(TestContext.Current.CancellationToken);
Assert.True(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop.jpg")));
Assert.False(File.Exists(Path.Combine(_dir, "metadata/movie/backdrop1.jpg")));
using (var ctx = CreateDbContext())
{
Assert.Equal("%MetadataPath%/movie/backdrop.jpg", Assert.Single(ctx.BaseItemImageInfos).Path);
}
}
[Fact]
public async Task Perform_IdenticalBackdropsOutsideMetadataFolder_KeepsFiles()
{
var movie = new BaseItemEntity { Id = Guid.NewGuid(), Type = typeof(Movie).FullName! };
using (var ctx = CreateDbContext())
{
ctx.BaseItems.Add(movie);
ctx.BaseItemImageInfos.AddRange(
Image(movie, WriteImage("media/movie/fanart.jpg", _image)),
Image(movie, WriteImage("media/movie/backdrop1.jpg", _image)));
await ctx.SaveChangesAsync(TestContext.Current.CancellationToken);
}
await CreateMigration().PerformAsync(TestContext.Current.CancellationToken);
Assert.True(File.Exists(Path.Combine(_dir, "media/movie/fanart.jpg")));
Assert.True(File.Exists(Path.Combine(_dir, "media/movie/backdrop1.jpg")));
using (var ctx = CreateDbContext())
{
Assert.Equal(2, ctx.BaseItemImageInfos.Count());
}
}
private string WriteImage(string relativePath, byte[] content)
{
var path = Path.Combine(_dir, relativePath);
File.WriteAllBytes(path, content);
return path;
}
private static BaseItemImageInfo Image(BaseItemEntity item, string path, ImageInfoImageType type = ImageInfoImageType.Backdrop)
=> new()
{
Id = Guid.NewGuid(),
ItemId = item.Id,
Item = item,
Path = path,
ImageType = type
};
private JellyfinDbContext CreateDbContext() => new(
_dbOptions,
NullLogger<JellyfinDbContext>.Instance,
new SqliteDatabaseProvider(new Mock<IApplicationPaths>().Object, NullLogger<SqliteDatabaseProvider>.Instance),
new NoLockBehavior(NullLogger<NoLockBehavior>.Instance));
private RemoveDuplicateBackdrops CreateMigration()
{
var factory = new Mock<IDbContextFactory<JellyfinDbContext>>();
factory.Setup(f => f.CreateDbContextAsync(It.IsAny<CancellationToken>())).ReturnsAsync(CreateDbContext);
var metadataPath = Path.Combine(_dir, "metadata");
var appHost = new Mock<IServerApplicationHost>();
appHost.Setup(h => h.ExpandVirtualPath(It.IsAny<string>()))
.Returns((string path) => path.Replace("%MetadataPath%", metadataPath, StringComparison.Ordinal));
var appPaths = new Mock<IServerApplicationPaths>();
appPaths.Setup(p => p.InternalMetadataPath).Returns(metadataPath);
var fileSystem = new ManagedFileSystem(NullLogger<ManagedFileSystem>.Instance, Mock.Of<IApplicationPaths>(), []);
return new RemoveDuplicateBackdrops(
new StartupLogger<RemoveDuplicateBackdrops>(NullLogger<RemoveDuplicateBackdrops>.Instance),
factory.Object,
appHost.Object,
appPaths.Object,
fileSystem);
}
}