Backport pull request #17930 from jellyfin/release-12.z

Fix playlist encoding recognition

Original-merge: e5d91342b7

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Cody Robibero <cody@robibe.ro>

MediaBrowser.Providers/MediaBrowser.Providers.csproj

@@ -26,6 +26,7 @@
     <PackageReference Include="SharpCompress" />
     <PackageReference Include="z440.atl.core" />
     <PackageReference Include="TMDbLib" />
+    <PackageReference Include="UTF.Unknown" />
   </ItemGroup>
 
   <PropertyGroup>

MediaBrowser.Providers/Playlists/PlaylistItemsProvider.cs

@@ -4,6 +4,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Jellyfin.Data.Enums;
@@ -15,6 +16,7 @@ using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.IO;
 using Microsoft.Extensions.Logging;
 using PlaylistsNET.Content;
+using UtfUnknown;
 
 namespace MediaBrowser.Providers.Playlists;
 
@@ -26,6 +28,11 @@ public class PlaylistItemsProvider : ILocalMetadataProvider<Playlist>,
     IForcedProvider,
     IHasItemChangeMonitor
 {
+    /// <summary>
+    /// Minimum confidence required before a detected encoding is preferred over UTF-8.
+    /// </summary>
+    private const float MinimumEncodingConfidence = 0.5f;
+
     private readonly IFileSystem _fileSystem;
     private readonly ILibraryManager _libraryManager;
     private readonly ILogger<PlaylistItemsProvider> _logger;
@@ -136,23 +143,38 @@ public class PlaylistItemsProvider : ILocalMetadataProvider<Playlist>,
     private IEnumerable<LinkedChild> GetPlsItems(Stream stream, string playlistPath, List<string> libraryRoots)
     {
         var content = new PlsContent();
-        var playlist = content.GetFromStream(stream);
+        var playlist = content.GetFromStream(stream, DetectEncoding(stream, playlistPath));
 
         return playlist.PlaylistEntries
                 .Select(i => GetLinkedChild(i.Path, playlistPath, libraryRoots))
                 .Where(i => i is not null);
     }
 
-    private IEnumerable<LinkedChild> GetM3uItems(Stream stream, string playlistPath, List<string> libraryRoots)
+    internal IEnumerable<LinkedChild> GetM3uItems(Stream stream, string playlistPath, List<string> libraryRoots)
     {
         var content = new M3uContent();
-        var playlist = content.GetFromStream(stream);
+        var playlist = content.GetFromStream(stream, DetectEncoding(stream, playlistPath));
 
         return playlist.PlaylistEntries
                 .Select(i => GetLinkedChild(i.Path, playlistPath, libraryRoots))
                 .Where(i => i is not null);
     }
 
+    private Encoding DetectEncoding(Stream stream, string playlistPath)
+    {
+        var detected = CharsetDetector.DetectFromStream(stream).Detected;
+        stream.Seek(0, SeekOrigin.Begin);
+
+        if (detected?.Encoding is null || detected.Confidence < MinimumEncodingConfidence)
+        {
+            _logger.LogDebug("Could not detect the encoding of playlist {Path}, assuming UTF-8", playlistPath);
+            return Encoding.UTF8;
+        }
+
+        _logger.LogDebug("Detected encoding {Encoding} for playlist {Path}", detected.Encoding.WebName, playlistPath);
+        return detected.Encoding;
+    }
+
     private IEnumerable<LinkedChild> GetZplItems(Stream stream, string playlistPath, List<string> libraryRoots)
     {
         var content = new ZplContent();
@@ -191,7 +213,7 @@ public class PlaylistItemsProvider : ILocalMetadataProvider<Playlist>,
     {
         item = null;
         string pathToCheck = _fileSystem.MakeAbsolutePath(Path.GetDirectoryName(playlistPath), itemPath);
-        if (!File.Exists(pathToCheck))
+        if (!File.Exists(pathToCheck) && !TryNormalizePath(ref pathToCheck))
         {
             return false;
         }
@@ -208,6 +230,36 @@ public class PlaylistItemsProvider : ILocalMetadataProvider<Playlist>,
         return false;
     }
 
+    private static bool TryNormalizePath(ref string path)
+    {
+        foreach (var form in new[] { NormalizationForm.FormC, NormalizationForm.FormD })
+        {
+            string normalized;
+            try
+            {
+                if (path.IsNormalized(form))
+                {
+                    continue;
+                }
+
+                normalized = path.Normalize(form);
+            }
+            catch (ArgumentException)
+            {
+                // The path is not valid Unicode, there is nothing to normalize.
+                return false;
+            }
+
+            if (File.Exists(normalized))
+            {
+                path = normalized;
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /// <inheritdoc />
     public bool HasChanged(BaseItem item, IDirectoryService directoryService)
     {

tests/Jellyfin.Providers.Tests/Playlists/PlaylistItemsProviderTests.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using MediaBrowser.Controller.Entities.Audio;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.IO;
+using MediaBrowser.Providers.Playlists;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Xunit;
+
+namespace Jellyfin.Providers.Tests.Playlists;
+
+public sealed class PlaylistItemsProviderTests : IDisposable
+{
+    private const string AccentedFolder = "Música épica";
+    private const string AccentedSong = "Canción.mp3";
+    private const string AsciiSong = "Song.mp3";
+
+    private readonly string _libraryRoot;
+    private readonly PlaylistItemsProvider _sut;
+
+    public PlaylistItemsProviderTests()
+    {
+        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+
+        _libraryRoot = Path.Combine(Path.GetTempPath(), "jellyfin-playlist-tests", Guid.NewGuid().ToString("N"));
+        var mediaFolder = Path.Combine(_libraryRoot, AccentedFolder);
+        Directory.CreateDirectory(mediaFolder);
+        File.WriteAllText(Path.Combine(mediaFolder, AccentedSong), string.Empty);
+        File.WriteAllText(Path.Combine(mediaFolder, AsciiSong), string.Empty);
+
+        var libraryManager = new Mock<ILibraryManager>();
+        libraryManager.Setup(m => m.FindByPath(It.IsAny<string>(), It.IsAny<bool?>()))
+            .Returns((string path, bool? _) => new Audio { Id = Guid.NewGuid(), Path = path });
+
+        var fileSystem = new Mock<IFileSystem>();
+        fileSystem.Setup(m => m.MakeAbsolutePath(It.IsAny<string>(), It.IsAny<string>()))
+            .Returns((string folderPath, string filePath) => Path.GetFullPath(Path.Combine(folderPath, filePath)));
+
+        _sut = new PlaylistItemsProvider(NullLogger<PlaylistItemsProvider>.Instance, libraryManager.Object, fileSystem.Object);
+    }
+
+    [Fact]
+    public void GetM3uItems_Utf8Entries_ResolvesAccentedPaths()
+        => AssertResolved(Encoding.UTF8, [$"{AccentedFolder}/{AccentedSong}"], 1);
+
+    /// <summary>
+    /// Playlists written by Windows media players default to the local codepage rather than UTF-8.
+    /// Decoding those as UTF-8 mangles every accented character and loses the entry.
+    /// </summary>
+    [Fact]
+    public void GetM3uItems_LegacyCodepageEntries_ResolvesAccentedPaths()
+        => AssertResolved(Encoding.GetEncoding(1252), [$"{AccentedFolder}/{AccentedSong}"], 1);
+
+    /// <summary>
+    /// The files on disk are stored precomposed (NFC), the playlist references them decomposed (NFD).
+    /// </summary>
+    [Fact]
+    public void GetM3uItems_DecomposedEntries_ResolvesAccentedPaths()
+        => AssertResolved(Encoding.UTF8, [$"{AccentedFolder}/{AccentedSong}".Normalize(NormalizationForm.FormD)], 1);
+
+    [Fact]
+    public void GetM3uItems_AsciiEntries_ResolvesPaths()
+        => AssertResolved(Encoding.UTF8, [$"{AccentedFolder}/{AsciiSong}"], 1);
+
+    [Fact]
+    public void GetM3uItems_Utf16Entries_ResolvesAccentedPaths()
+        => AssertResolved(Encoding.Unicode, [$"{AccentedFolder}/{AccentedSong}"], 1);
+
+    [Fact]
+    public void GetM3uItems_MixedEntries_ResolvesEveryEntry()
+        => AssertResolved(
+            Encoding.GetEncoding(1252),
+            [$"{AccentedFolder}/{AccentedSong}", $"{AccentedFolder}/{AsciiSong}"],
+            2);
+
+    [Fact]
+    public void GetM3uItems_MissingFile_ResolvesNothing()
+        => AssertResolved(Encoding.UTF8, [$"{AccentedFolder}/Does not exist.mp3"], 0);
+
+    public void Dispose()
+    {
+        if (Directory.Exists(_libraryRoot))
+        {
+            Directory.Delete(_libraryRoot, true);
+        }
+    }
+
+    private void AssertResolved(Encoding encoding, string[] entries, int expected)
+    {
+        var playlistPath = Path.Combine(_libraryRoot, "playlist.m3u");
+        var content = new StringBuilder("#EXTM3U\n");
+        foreach (var entry in entries)
+        {
+            content.Append("#EXTINF:1,Title\n").Append(entry).Append('\n');
+        }
+
+        File.WriteAllBytes(
+            playlistPath,
+            [.. encoding.GetPreamble(), .. encoding.GetBytes(content.ToString())]);
+
+        using var stream = File.OpenRead(playlistPath);
+        var resolved = _sut.GetM3uItems(stream, playlistPath, [_libraryRoot]).ToList();
+
+        Assert.Equal(expected, resolved.Count);
+    }
+}