diff --git a/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackController.java b/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackController.java
index 2907d8195..01c6405a7 100644
--- a/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackController.java
+++ b/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackController.java
@@ -650,12 +650,17 @@ public class PlaybackController implements PlaybackControllerNotifiable {
                 // User explicitly disabled subtitles
                 mCurrentOptions.setSubtitleStreamIndex(null);
             } else if (response.getMediaSource().getMediaStreams() != null) {
-                // Find subtitle stream matching saved language
+                // Find subtitle stream matching saved language, keeping the requested stream if it matches
+                // (e.g. when playback restarts to switch between baked and non-baked subtitles)
+                Integer requestedIndex = mCurrentOptions.getSubtitleStreamIndex();
                 Integer matchingIndex = null;
                 for (MediaStream stream : response.getMediaSource().getMediaStreams()) {
                     if (stream.getType() == MediaStreamType.SUBTITLE && lastSubtitleLanguage.equals(stream.getLanguage())) {
-                        matchingIndex = stream.getIndex();
-                        break;
+                        if (matchingIndex == null) matchingIndex = stream.getIndex();
+                        if (requestedIndex != null && requestedIndex == stream.getIndex()) {
+                            matchingIndex = requestedIndex;
+                            break;
+                        }
                     }
                 }
                 mCurrentOptions.setSubtitleStreamIndex(matchingIndex);
diff --git a/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackControllerHelper.kt b/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackControllerHelper.kt
index 2b8d2c543..ca68a6cd6 100644
--- a/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackControllerHelper.kt
+++ b/app/src/main/java/org/jellyfin/androidtv/ui/playback/PlaybackControllerHelper.kt
@@ -124,7 +124,15 @@ fun PlaybackController.setSubtitleIndex(index: Int, force: Boolean = false) {
 					// The server does not send a reliable index in all cases, so calculate it manually
 					val localIndex = mediaSource.mediaStreams.orEmpty()
 						.filter { it.type == MediaStreamType.SUBTITLE }
-						.filter { it.deliveryMethod == SubtitleDeliveryMethod.EMBED || it.deliveryMethod == SubtitleDeliveryMethod.HLS }
+						.filter {
+							if (stream.deliveryMethod == SubtitleDeliveryMethod.EMBED) {
+								// ExoPlayer exposes every subtitle track of the container, including the ones the server
+								// would burn in (ASS/SSA without libass), so they must be counted to match its track groups
+								!it.isExternal
+							} else {
+								it.deliveryMethod == SubtitleDeliveryMethod.EMBED || it.deliveryMethod == SubtitleDeliveryMethod.HLS
+							}
+						}
 						.indexOf(stream)
 						.takeIf { it != -1 }