Skip to content

Commit

Permalink
Fix and rewrite audio and subtitle track selection when transcoding
Browse files Browse the repository at this point in the history
  • Loading branch information
@Maxr1998
Maxr1998 committed Dec 21, 2022
1 parent 652dde3 commit 00f3b89
Show file tree
Hide file tree
Showing 7 changed files with 181 additions and 96 deletions.
56 changes: 35 additions & 21 deletions app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import com.google.android.exoplayer2.analytics.DefaultAnalyticsCollector
import com.google.android.exoplayer2.mediacodec.MediaCodecDecoderException
import com.google.android.exoplayer2.mediacodec.MediaCodecInfo
import com.google.android.exoplayer2.mediacodec.MediaCodecSelector
import com.google.android.exoplayer2.trackselection.DefaultTrackSelector
import com.google.android.exoplayer2.util.Clock
import com.google.android.exoplayer2.util.EventLogger
import com.google.android.exoplayer2.util.MimeTypes
Expand All @@ -40,6 +41,7 @@ import org.jellyfin.mobile.player.queue.QueueManager
import org.jellyfin.mobile.player.source.JellyfinMediaSource
import org.jellyfin.mobile.player.ui.DecoderType
import org.jellyfin.mobile.player.ui.DisplayPreferences
import org.jellyfin.mobile.player.ui.PlayState
import org.jellyfin.mobile.utils.Constants
import org.jellyfin.mobile.utils.Constants.SUPPORTED_VIDEO_PLAYER_PLAYBACK_ACTIONS
import org.jellyfin.mobile.utils.applyDefaultAudioAttributes
Expand Down Expand Up @@ -84,7 +86,8 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
val notificationHelper: PlayerNotificationHelper by lazy { PlayerNotificationHelper(this) }

// Media source handling
val mediaQueueManager = QueueManager(this)
private val trackSelector = DefaultTrackSelector(getApplication())
val mediaQueueManager = QueueManager(this, trackSelector)
val mediaSourceOrNull: JellyfinMediaSource?
get() = mediaQueueManager.mediaQueue.value?.jellyfinMediaSource

Expand Down Expand Up @@ -213,7 +216,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
}
_player.value = ExoPlayer.Builder(getApplication(), renderersFactory, get()).apply {
setUsePlatformDiagnostics(false)
setTrackSelector(mediaQueueManager.trackSelector)
setTrackSelector(trackSelector)
setAnalyticsCollector(analyticsCollector)
}.build().apply {
addListener(this@PlayerViewModel)
Expand Down Expand Up @@ -365,18 +368,22 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
}

// Stop active encoding if transcoding
if (mediaSource.playMethod == PlayMethod.TRANSCODE) {
hlsSegmentApi.stopEncodingProcess(
deviceId = apiClient.deviceInfo.id,
playSessionId = mediaSource.playSessionId,
)
}
stopTranscoding(mediaSource)
} catch (e: ApiClientException) {
Timber.e(e, "Failed to report playback stop")
}
}
}

suspend fun stopTranscoding(mediaSource: JellyfinMediaSource) {
if (mediaSource.playMethod == PlayMethod.TRANSCODE) {
hlsSegmentApi.stopEncodingProcess(
deviceId = apiClient.deviceInfo.id,
playSessionId = mediaSource.playSessionId,
)
}
}

// Player controls

fun play() {
Expand Down Expand Up @@ -418,29 +425,36 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application),
}
}

fun getStateAndPause(): PlayState? {
val player = playerOrNull ?: return null

val playWhenReady = player.playWhenReady
player.pause()
val position = player.contentPosition

return PlayState(playWhenReady, position)
}

/**
* @see QueueManager.selectAudioTrack
*/
fun selectAudioTrack(streamIndex: Int): Boolean = mediaQueueManager.selectAudioTrack(streamIndex).also { success ->
if (success) playerOrNull?.logTracks(analyticsCollector)
suspend fun selectAudioTrack(streamIndex: Int): Boolean {
return mediaQueueManager.selectAudioTrack(streamIndex).also { success ->
if (success) playerOrNull?.logTracks(analyticsCollector)
}
}

/**
* @see QueueManager.selectSubtitle
*/
fun selectSubtitle(streamIndex: Int): Boolean = mediaQueueManager.selectSubtitle(streamIndex).also { success ->
if (success) playerOrNull?.logTracks(analyticsCollector)
suspend fun selectSubtitle(streamIndex: Int): Boolean {
return mediaQueueManager.selectSubtitle(streamIndex).also { success ->
if (success) playerOrNull?.logTracks(analyticsCollector)
}
}

fun changeBitrate(bitrate: Int?) {
val player = playerOrNull ?: return

val playWhenReady = player.playWhenReady
pause()
val position = player.contentPosition
viewModelScope.launch {
mediaQueueManager.changeBitrate(bitrate, position, playWhenReady)
}
suspend fun changeBitrate(bitrate: Int?): Boolean {
return mediaQueueManager.changeBitrate(bitrate)
}

/**
Expand Down
127 changes: 87 additions & 40 deletions app/src/main/java/org/jellyfin/mobile/player/queue/QueueManager.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.jellyfin.mobile.player.queue

import android.app.Application
import android.net.Uri
import androidx.annotation.CheckResult
import androidx.lifecycle.LiveData
Expand Down Expand Up @@ -35,16 +34,19 @@ import java.util.UUID

class QueueManager(
private val viewModel: PlayerViewModel,
private val trackSelector: DefaultTrackSelector,
) : KoinComponent {
private val apiClient: ApiClient = get()
private val videosApi: VideosApi = apiClient.videosApi
private val mediaSourceResolver: MediaSourceResolver by inject()
private val deviceProfileBuilder: DeviceProfileBuilder by inject()
private val videosApi: VideosApi = apiClient.videosApi
val trackSelector = DefaultTrackSelector(viewModel.getApplication<Application>())
private val deviceProfile = deviceProfileBuilder.getDeviceProfile()
private val _mediaQueue: MutableLiveData<QueueItem.Loaded> = MutableLiveData()
val mediaQueue: LiveData<QueueItem.Loaded> get() = _mediaQueue

private var deviceProfile = deviceProfileBuilder.getDeviceProfile()
private val currentMediaSource: JellyfinMediaSource?
get() = _mediaQueue.value?.jellyfinMediaSource

private var currentPlayOptions: PlayOptions? = null

/**
Expand All @@ -64,10 +66,17 @@ class QueueManager(
audioStreamIndex = playOptions.audioStreamIndex,
subtitleStreamIndex = playOptions.subtitleStreamIndex,
).onSuccess { jellyfinMediaSource ->
// Ensure transcoding of the current element is stopped
currentMediaSource?.let { oldMediaSource ->
viewModel.stopTranscoding(oldMediaSource)
}

// Apply new queue
val previous = QueueItem.Stub(playOptions.ids.take(playOptions.startIndex))
val next = QueueItem.Stub(playOptions.ids.drop(playOptions.startIndex + 1))
createQueueItem(jellyfinMediaSource, previous, next).play(playWhenReady)
val new = createQueueItem(jellyfinMediaSource, previous, next)
currentPlayOptions = playOptions
new.play(playWhenReady)
}.onFailure { error ->
// Should always be of this type, other errors are silently dropped
return error as? PlayerException
Expand All @@ -85,21 +94,20 @@ class QueueManager(

/**
* Change the maximum bitrate to the specified value.
*
* @param positionMs the current stream position to return to after reloading the source.
* @param playWhenReady whether the stream should play after the source was reloaded.
*/
suspend fun changeBitrate(bitrate: Int?, positionMs: Long, playWhenReady: Boolean) {
val currentPlayOptions = currentPlayOptions ?: return
suspend fun changeBitrate(bitrate: Int?): Boolean {
val currentPlayOptions = currentPlayOptions ?: return false

// Bitrate didn't change, ignore
if (currentPlayOptions.maxBitrate == bitrate) return
if (currentPlayOptions.maxBitrate == bitrate) return true

val currentPlayState = viewModel.getStateAndPause() ?: return false

val playOptions = currentPlayOptions.copy(
startPositionTicks = positionMs * Constants.TICKS_PER_MILLISECOND,
startPositionTicks = currentPlayState.position * Constants.TICKS_PER_MILLISECOND,
maxBitrate = bitrate,
)
startPlayback(playOptions, playWhenReady)
return startPlayback(playOptions, currentPlayState.playWhenReady) == null
}

@CheckResult
Expand Down Expand Up @@ -247,10 +255,9 @@ class QueueManager(
}

fun selectInitialTracks() {
val queueItem = _mediaQueue.value ?: return
val mediaSource = queueItem.jellyfinMediaSource
selectAudioTrack(mediaSource.selectedAudioStream?.index ?: -1, initial = true)
selectSubtitle(mediaSource.selectedSubtitleStream?.index ?: -1, initial = true)
val mediaSource = currentMediaSource ?: return
selectAudioTrack(mediaSource, mediaSource.selectedAudioStream?.index ?: -1, initial = true)
selectSubtitle(mediaSource, mediaSource.selectedSubtitleStream?.index ?: -1, initial = true)
}

/**
Expand All @@ -259,27 +266,45 @@ class QueueManager(
* @param streamIndex the [MediaStream.index] that should be selected
* @return true if the audio track was changed
*/
fun selectAudioTrack(streamIndex: Int): Boolean {
return selectAudioTrack(streamIndex, initial = false)
suspend fun selectAudioTrack(streamIndex: Int): Boolean {
val mediaSource = currentMediaSource ?: return false

return when (mediaSource.playMethod) {
// For transcoding playback, special handling is required
PlayMethod.TRANSCODE -> {
val currentPlayOptions = currentPlayOptions ?: return false
val currentPlayState = viewModel.getStateAndPause() ?: return false

val playOptions = currentPlayOptions.copy(
startPositionTicks = currentPlayState.position * Constants.TICKS_PER_MILLISECOND,
audioStreamIndex = streamIndex,
)
startPlayback(playOptions, currentPlayState.playWhenReady)

// Player menus will be updated after playback started
false
}
else -> selectAudioTrack(mediaSource, streamIndex, initial = false)
}
}

/**
* @param initial whether this is an initial selection and checks for re-selection should be skipped.
* @see selectAudioTrack
*/
@Suppress("ReturnCount")
private fun selectAudioTrack(streamIndex: Int, initial: Boolean): Boolean {
val mediaSource = _mediaQueue.value?.jellyfinMediaSource ?: return false
val sourceIndex = mediaSource.audioStreams.binarySearchBy(streamIndex, selector = MediaStream::index)
private fun selectAudioTrack(mediaSource: JellyfinMediaSource, streamIndex: Int, initial: Boolean): Boolean {
if (mediaSource.playMethod == PlayMethod.TRANSCODE) {
return when {
initial -> true
else -> error("Selecting audio tracks in ExoPlayer isn't supported while transcoding")
}
}
val sourceIndex = mediaSource.audioStreams.indexOfFirst { stream -> stream.index == streamIndex }

when {
// Fast-pass: Skip execution on subsequent calls with the correct selection or if only one track exists
mediaSource.audioStreams.size == 1 || !initial && streamIndex == mediaSource.selectedAudioStream?.index -> return true
// For transcoded media, special handling is required
mediaSource.playMethod == PlayMethod.TRANSCODE -> {
// TODO: handle stream selection for transcodes (reinitialize media source)
return true
}
// Apply selection in media source, abort on failure
!mediaSource.selectAudioStream(sourceIndex) -> return false
}
Expand All @@ -293,17 +318,40 @@ class QueueManager(
* @param streamIndex the [MediaStream.index] that should be selected
* @return true if the subtitle was changed
*/
fun selectSubtitle(streamIndex: Int): Boolean {
return selectSubtitle(streamIndex, initial = false)
suspend fun selectSubtitle(streamIndex: Int): Boolean {
val mediaSource = currentMediaSource ?: return false

return when (mediaSource.playMethod) {
// For transcoding playback, special handling is required
PlayMethod.TRANSCODE -> {
val currentPlayOptions = currentPlayOptions ?: return false
val currentPlayState = viewModel.getStateAndPause() ?: return false

val playOptions = currentPlayOptions.copy(
startPositionTicks = currentPlayState.position * Constants.TICKS_PER_MILLISECOND,
subtitleStreamIndex = streamIndex,
)
startPlayback(playOptions, currentPlayState.playWhenReady)

// Player menus will be updated after playback started
false
}
else -> selectSubtitle(mediaSource, streamIndex, initial = false)
}
}

/**
* @param initial whether this is an initial selection and checks for re-selection should be skipped.
* @see selectSubtitle
*/
private fun selectSubtitle(streamIndex: Int, initial: Boolean): Boolean {
val mediaSource = _mediaQueue.value?.jellyfinMediaSource ?: return false
val sourceIndex = mediaSource.subtitleStreams.binarySearchBy(streamIndex, selector = MediaStream::index)
private fun selectSubtitle(mediaSource: JellyfinMediaSource, streamIndex: Int, initial: Boolean): Boolean {
if (mediaSource.playMethod == PlayMethod.TRANSCODE) {
return when {
initial -> true
else -> error("Selecting subtitle tracks in ExoPlayer isn't supported while transcoding")
}
}
val sourceIndex = mediaSource.subtitleStreams.indexOfFirst { stream -> stream.index == streamIndex }

when {
// Fast-pass: Skip execution on subsequent calls with the correct selection
Expand All @@ -325,14 +373,13 @@ class QueueManager(
*
* @return true if subtitles are enabled now, false if not
*/
fun toggleSubtitles(): Boolean {
val mediaSource = _mediaQueue.value?.jellyfinMediaSource ?: return false
selectSubtitle(
when (mediaSource.selectedSubtitleStream) {
null -> mediaSource.subtitleStreams.firstOrNull()?.index ?: -1
else -> -1
},
)
suspend fun toggleSubtitles(): Boolean {
val mediaSource = currentMediaSource ?: return false
val newSubtitleIndex = when (mediaSource.selectedSubtitleStream) {
null -> mediaSource.subtitleStreams.firstOrNull()?.index ?: -1
else -> -1
}
selectSubtitle(newSubtitleIndex)
return mediaSource.selectedSubtitleStream != null
}

Expand Down
4 changes: 4 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/player/source/MediaSourceResolver.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ class MediaSourceResolver(private val apiClient: ApiClient) {
itemId = itemId,
data = PlaybackInfoDto(
userId = apiClient.userId,
// We need to remove the dashes so that the server can find the correct media source.
// And if we didn't pass the mediaSourceId, our stream indices would silently get ignored.
// https://github.com/jellyfin/jellyfin/blob/9a35fd673203cfaf0098138b2768750f4818b3ab/Jellyfin.Api/Helpers/MediaInfoHelper.cs#L196-L201
mediaSourceId = itemId.toString().replace("-", ""),
deviceProfile = deviceProfile,
maxStreamingBitrate = maxStreamingBitrate ?: deviceProfile.maxStreamingBitrate,
startTimeTicks = startTimeTicks,
Expand Down
6 changes: 6 additions & 0 deletions app/src/main/java/org/jellyfin/mobile/player/ui/PlayState.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package org.jellyfin.mobile.player.ui

data class PlayState(
val playWhenReady: Boolean,
val position: Long,
)
Loading

0 comments on commit 00f3b89

Please sign in to comment.