fix(ios): stream formats AVPlayer can play

AVPlayer can't open WebM/Ogg or DASH, so the default opus/webm stream failed
silently on iOS.

- Rank streams the platform's player can play above format/quality prefs
- Default iOS streaming to aac/mp4, migrate saved WebM/Ogg prefs on iOS and
  hide unplayable formats in the picker (desktop/Android keep opus/webm)
- Normalize codec/container names on iOS like androidJvm (mp4a -> aac, m4a -> mp4)
- Wait up to 30s for the audio plugin to load when resolving, so the queue
  restored at startup isn't failed permanently before plugins are ready
This commit is contained in:
Kingkor Roy Tirtho 2026-10-26 13:40:00 +06:00
parent 99ba08501b
commit cb8316d141
10 changed files with 198 additions and 28 deletions

View File

@ -162,14 +162,6 @@ actual class NewPipeService {
* "aac"/"mp4". Without this a real AAC stream never matched the preference, and the hard-coded
* aac/mp4 DASH/HLS manifests always won instead.
*/
private fun normalizeCodec(codec: String?): String {
val value = codec.orEmpty()
return if (value.startsWith("mp4a", ignoreCase = true)) "aac" else value
}
private fun normalizeContainer(suffix: String): String =
if (suffix.equals("m4a", ignoreCase = true)) "mp4" else suffix
private fun DeliveryMethod.toProtocol(): StreamProtocol = when (this) {
DeliveryMethod.DASH -> StreamProtocol.DASH
DeliveryMethod.HLS -> StreamProtocol.HLS

View File

@ -49,4 +49,14 @@ data class MediaStream(
val container: String,
val bitrate: Int,
val isAudio: Boolean,
val isVideo: Boolean)
val isVideo: Boolean)
/** Maps NewPipe's codec names onto the plugin's AudioFormat names (mp4a.40.2 → aac). */
internal fun normalizeCodec(codec: String?): String {
val value = codec.orEmpty()
return if (value.startsWith("mp4a", ignoreCase = true)) "aac" else value
}
/** Maps NewPipe's file suffixes onto the plugin's AudioFormat containers (m4a → mp4). */
internal fun normalizeContainer(suffix: String): String =
if (suffix.equals("m4a", ignoreCase = true)) "mp4" else suffix

View File

@ -23,13 +23,18 @@ import org.spotube.plugin_interfaces.plugin_apis.audio.AudioQuality
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioStream
import org.spotube.plugin_interfaces.plugin_apis.audio.StreamProtocol
import org.spotube.plugin_interfaces.plugin_apis.metadata.track.MetadataTrack
import org.spotube.spotube.PlatformType
import org.spotube.spotube.core.audioplayer.AudioPlayerQueue
import org.spotube.spotube.core.audioplayer.QueueEntry
import org.spotube.spotube.core.di.injectLogger
import org.spotube.spotube.getPlatform
import org.spotube.spotube.modules.plugin.AudioPluginSource
import org.spotube.spotube.modules.settings.UserSettingsSource
import kotlinx.coroutines.flow.filterNotNull
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeoutOrNull
import org.koin.core.component.KoinComponent
import kotlin.math.abs
import kotlin.time.Clock
@ -111,7 +116,14 @@ class StreamingUrlRepository(
// read it, nor replace it with a stream playback didn't ask for.
val useUrlCache = request == null
// A built-in audio plugin is always selected, so null means plugins are still loading.
// The player requests the restored queue's track right at startup; failing that request
// leaves the item permanently failed (AVPlayer), so wait for the plugin instead.
val audioPlugin = pluginManager.selectedAudioPlugin.value
?: withTimeoutOrNull(AUDIO_PLUGIN_LOAD_TIMEOUT) {
logger.i { "[Resolve $trackId] waiting for the audio plugin to load" }
pluginManager.selectedAudioPlugin.filterNotNull().first()
}
if (audioPlugin == null) {
logger.w { "No audio plugin selected while resolving stream for track $trackId" }
return null
@ -243,10 +255,12 @@ class StreamingUrlRepository(
logger.i {
"[Resolve $trackId] selecting from ${streams.size} candidates ($origin), " +
"want ${preferredFormat.codec}/${preferredFormat.container} $preferredQuality " +
"(rank key = formatMismatch, qualityTypeMismatch, qualityDistance, channelDistance; lowest wins):\n" +
"(rank key = unplayable, formatMismatch, qualityTypeMismatch, qualityDistance, channelDistance; " +
"lowest wins):\n" +
streams.joinToString("\n") { stream ->
val marker = if (stream === selected) " -> " else " "
"$marker${stream.describeForLog()} key=(${formatMismatchCount(stream, preferredFormat)}, " +
"$marker${stream.describeForLog()} key=(${if (stream.isPlayableOn(currentPlatform)) 0 else 1}, " +
"${formatMismatchCount(stream, preferredFormat)}, " +
"${if (stream.matchesQualityType(preferredQuality)) 0 else 1}, " +
"${stream.qualityDistance(preferredQuality).let { if (it == Long.MAX_VALUE) "inf" else "$it" }}, " +
"${stream.channelDistance(preferredQuality)}) ${urlSummary(stream.url)}"
@ -254,6 +268,10 @@ class StreamingUrlRepository(
}
when {
selected == null -> logger.w { "[Resolve $trackId] no stream selected" }
!selected.isPlayableOn(currentPlatform) -> logger.w {
"[Resolve $trackId] NO PLAYABLE STREAM on $currentPlatform; falling back to " +
"${selected.describeForLog()}, which the player will likely reject"
}
formatMismatchCount(selected, preferredFormat) > 0 -> logger.w {
"[Resolve $trackId] FORMAT FALLBACK: wanted ${preferredFormat.codec}/${preferredFormat.container}, " +
"got ${selected.codec}/${selected.container} (${selected.protocol})"
@ -371,6 +389,8 @@ class StreamingUrlRepository(
* Resolves a stream URL to an absolute URL. Relative manifest paths (DASH/HLS) are
* resolved against the YouTube origin; progressive URLs are returned unchanged.
*/
private val AUDIO_PLUGIN_LOAD_TIMEOUT = 30.seconds
internal fun normalizeManifestUrl(url: String, protocol: StreamProtocol): String {
if (url.startsWith("http")) return url
return when (protocol) {
@ -379,12 +399,21 @@ internal fun normalizeManifestUrl(url: String, protocol: StreamProtocol): String
}
}
/** Selects the available stream closest to the user's preferred format and quality. */
internal val currentPlatform: PlatformType by lazy { getPlatform().type }
/**
* Selects the available stream closest to the user's preferred format and quality.
* Streams the [platform]'s player can decode always win over ones it can't, whatever the
* preferences say: e.g. opus/webm is unplayable on iOS.
*/
internal fun selectPreferredAudioStream(
streams: List<AudioStream>,
preferredFormat: AudioFormat,
preferredQuality: AudioQuality): AudioStream? = streams.minWithOrNull(
preferredQuality: AudioQuality,
platform: PlatformType = currentPlatform): AudioStream? = streams.minWithOrNull(
compareBy<AudioStream> { stream ->
if (stream.isPlayableOn(platform)) 0 else 1
}.thenBy { stream ->
formatMismatchCount(stream, preferredFormat)
}.thenBy { stream ->
if (stream.matchesQualityType(preferredQuality)) 0 else 1
@ -395,6 +424,21 @@ internal fun selectPreferredAudioStream(
}
)
// AVPlayer has no WebM/Ogg/Matroska demuxer, so e.g. YouTube's opus/webm streams fail to load.
private val IOS_UNSUPPORTED_CONTAINERS = setOf("webm", "ogg", "oga", "mkv", "matroska")
/**
* Whether [platform]'s player can play this stream. VLC (desktop) and ExoPlayer (Android)
* handle everything plugins return; AVPlayer (iOS) can't demux WebM/Ogg and can't play DASH.
*/
internal fun AudioStream.isPlayableOn(platform: PlatformType): Boolean =
isContainerPlayableOn(container, platform) &&
!(platform == PlatformType.IOS && protocol == StreamProtocol.DASH)
/** Whether [platform]'s player can open [container] at all; see [AudioStream.isPlayableOn]. */
internal fun isContainerPlayableOn(container: String, platform: PlatformType): Boolean =
platform != PlatformType.IOS || container.lowercase() !in IOS_UNSUPPORTED_CONTAINERS
private fun formatMismatchCount(stream: AudioStream, preferredFormat: AudioFormat): Int =
(if (stream.codec.equals(preferredFormat.codec, ignoreCase = true)) 0 else 1) +
(if (stream.container.equals(preferredFormat.container, ignoreCase = true)) 0 else 1)

View File

@ -17,6 +17,9 @@
package org.spotube.spotube.modules.settings
import org.spotube.spotube.core.server.isContainerPlayableOn
import org.spotube.spotube.core.server.currentPlatform
import org.spotube.spotube.PlatformType
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioFormat
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioQuality
import kotlinx.serialization.SerialName
@ -42,15 +45,7 @@ data class UserSettings(
val palette: MillenniumPalette = MillenniumPalette.Blue,
// Playback
val streamingMusicFormat: AudioFormat = AudioFormat(
codec = "opus",
container = "webm",
qualities = listOf(
AudioQuality.Lossy(bitrate = 44_000),
AudioQuality.Lossy(bitrate = 96_000),
AudioQuality.Lossy(bitrate = 128_000),
AudioQuality.Lossy(bitrate = 256_000))
),
val streamingMusicFormat: AudioFormat = defaultStreamingMusicFormat(),
val streamingMusicQuality: AudioQuality = AudioQuality.Lossy(bitrate = 256_000),
val enableMusicCaching: Boolean = true,
val cacheFolder: String? = null,
@ -111,3 +106,24 @@ data class JamBroker(
val clientIdPrefix: String = "spotube",
val keepAliveSeconds: Int = 30,
val connectionTimeoutSeconds: Int = 10)
/**
* opus/webm everywhere except iOS, whose AVPlayer can't open WebM/Ogg, so it streams AAC.
* TODO: revisit once iOS can play Opus (e.g. remuxing to CAF or a non-AVPlayer backend).
*/
internal fun defaultStreamingMusicFormat(platform: PlatformType = currentPlatform): AudioFormat {
val ios = platform == PlatformType.IOS
return AudioFormat(
codec = if (ios) "aac" else "opus",
container = if (ios) "mp4" else "webm",
qualities = listOf(
AudioQuality.Lossy(bitrate = 44_000),
AudioQuality.Lossy(bitrate = 96_000),
AudioQuality.Lossy(bitrate = 128_000),
AudioQuality.Lossy(bitrate = 256_000)))
}
/** Moves settings saved with a streaming format [platform] can't play (iOS' old opus/webm default) to the default. */
internal fun UserSettings.withPlayableStreamingFormat(platform: PlatformType = currentPlatform): UserSettings =
if (isContainerPlayableOn(streamingMusicFormat.container, platform)) this
else copy(streamingMusicFormat = defaultStreamingMusicFormat(platform))

View File

@ -54,7 +54,7 @@ class SettingsRepository(private val database: Database) : UserSettingsSource {
override val userSettings: StateFlow<UserSettings> = database.settingsDataStore.data.map { prefs ->
val json = prefs[SETTINGS_KEY]
if (json != null) {
settingsJson.decodeFromString<UserSettings>(json as String)
settingsJson.decodeFromString<UserSettings>(json as String).withPlayableStreamingFormat()
} else {
UserSettings() // Default value
}
@ -73,6 +73,6 @@ class SettingsRepository(private val database: Database) : UserSettingsSource {
suspend fun getCurrentSettings(): UserSettings {
val preferences = database.settingsDataStore.data.first()
val storedJson = preferences[SETTINGS_KEY] ?: return UserSettings()
return settingsJson.decodeFromString<UserSettings>(storedJson)
return settingsJson.decodeFromString<UserSettings>(storedJson).withPlayableStreamingFormat()
}
}

View File

@ -18,6 +18,8 @@
package org.spotube.spotube.modules.settings.sections
import org.spotube.spotube.core.server.isContainerPlayableOn
import org.spotube.spotube.core.server.currentPlatform
import org.spotube.millennium.composables.MillenniumIcon
import org.spotube.millennium.composables.MillenniumSeparator
import org.spotube.millennium.theme.MillenniumTheme
@ -56,6 +58,7 @@ private val standardLossyQualities = listOf(
AudioQuality.Lossy(bitrate = 128_000),
AudioQuality.Lossy(bitrate = 256_000))
// Formats the platform's player can't open aren't offered (iOS: no WebM/Ogg).
internal val streamingFormatPresets = listOf(
AudioFormat(
codec = "opus",
@ -68,7 +71,8 @@ internal val streamingFormatPresets = listOf(
AudioFormat(
codec = "vorbis",
container = "ogg",
qualities = standardLossyQualities))
qualities = standardLossyQualities)
).filter { isContainerPlayableOn(it.container, currentPlatform) }
internal val downloadFormatPresets = listOf(
AudioFormat(

View File

@ -8,6 +8,8 @@ import org.spotube.plugin_interfaces.plugin_apis.metadata.track.MetadataTrack
import org.spotube.spotube.core.audioplayer.QueueEntry
import org.spotube.spotube.modules.settings.UserSettings
import co.touchlab.kermit.Logger
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.runTest
import org.koin.core.context.startKoin
import org.koin.core.context.stopKoin
@ -130,6 +132,22 @@ class StreamingUrlRepositoryBehaviorTest {
assertNull(info)
}
@Test
fun `waits for an audio plugin that is still loading`() = runTest {
val h = Harness()
val loader = launch {
delay(5_000)
h.selectPlugin("pluginA").audioAPI.onGetStreamsByTrack = {
listOf(streamedSource("src1", listOf(stream("https://example.com/a.opus"))))
}
}
val info = h.repository.resolveStreamInfo(track("t1"))
loader.join()
assertEquals("https://example.com/a.opus", info?.url)
}
@Test
fun `returns null when the plugin yields no sources`() = runTest {
val h = Harness()

View File

@ -1,5 +1,6 @@
package org.spotube.spotube.core.server
import org.spotube.spotube.PlatformType
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioFormat
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioQuality
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioStream
@ -13,6 +14,7 @@ import kotlin.test.assertSame
* Tests for the stream preference selection used by [StreamingUrlRepository].
*
* Ordering contract (best first):
* 0. Playable by the platform's player (iOS: no WebM/Ogg containers, no DASH).
* 1. Fewest codec/container mismatches against the preferred format.
* 2. Matching quality family (lossy vs lossless).
* 3. Closest bitrate / sample rate.
@ -54,7 +56,8 @@ class StreamingUrlRepositoryTest {
private fun select(
streams: List<AudioStream>,
format: AudioFormat = opusWebm(),
quality: AudioQuality = AudioQuality.Lossy(256_000)) = selectPreferredAudioStream(streams, format, quality)
quality: AudioQuality = AudioQuality.Lossy(256_000),
platform: PlatformType = PlatformType.Linux) = selectPreferredAudioStream(streams, format, quality, platform)
// ---------- empty / single ----------
@ -403,4 +406,34 @@ class StreamingUrlRepositoryTest {
assertEquals("https://www.youtube.com", normalizeManifestUrl("", StreamProtocol.DASH))
assertEquals("https://www.youtube.com", normalizeManifestUrl("", StreamProtocol.HLS))
}
// ---------- device playability ----------
@Test
fun `on iOS a playable m4a stream beats the preferred but unplayable opus webm`() {
val opus = lossy("opus-160", 160_000)
val aac = lossy("aac-128", 128_000, codec = "mp4a.40.2", container = "m4a")
assertSame(aac, select(listOf(opus, aac), platform = PlatformType.IOS))
}
@Test
fun `on iOS dash manifests are skipped in favour of progressive streams`() {
val dash = lossy("dash", 256_000, codec = "mp4a.40.2", container = "m4a", protocol = StreamProtocol.DASH)
val progressive = lossy("aac-128", 128_000, codec = "mp4a.40.2", container = "m4a")
assertSame(progressive, select(listOf(dash, progressive), platform = PlatformType.IOS))
}
@Test
fun `on iOS an unplayable stream is still returned when nothing else exists`() {
val opus = lossy("opus-160", 160_000)
assertSame(opus, select(listOf(opus), platform = PlatformType.IOS))
}
@Test
fun `off iOS the preferred opus webm stream still wins`() {
val opus = lossy("opus-160", 160_000)
val aac = lossy("aac-128", 128_000, codec = "mp4a.40.2", container = "m4a")
assertSame(opus, select(listOf(opus, aac), platform = PlatformType.Android))
}
}

View File

@ -0,0 +1,53 @@
/*
* Copyright (C) 2026 Kingkor Roy Tirtho and Spotube Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.spotube.spotube.modules.settings
import org.spotube.plugin_interfaces.plugin_apis.audio.AudioFormat
import org.spotube.spotube.PlatformType
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertSame
class StreamingFormatDefaultsTest {
private val opusWebm = AudioFormat(codec = "opus", container = "webm", qualities = emptyList())
@Test
fun `ios defaults to aac in mp4`() {
val format = defaultStreamingMusicFormat(PlatformType.IOS)
assertEquals("aac" to "mp4", format.codec to format.container)
}
@Test
fun `other platforms keep opus in webm`() {
val format = defaultStreamingMusicFormat(PlatformType.Android)
assertEquals("opus" to "webm", format.codec to format.container)
}
@Test
fun `saved opus webm is moved to aac on ios`() {
val migrated = UserSettings(streamingMusicFormat = opusWebm).withPlayableStreamingFormat(PlatformType.IOS)
assertEquals("mp4", migrated.streamingMusicFormat.container)
}
@Test
fun `saved opus webm is kept off ios`() {
val settings = UserSettings(streamingMusicFormat = opusWebm)
assertSame(settings, settings.withPlayableStreamingFormat(PlatformType.Linux))
}
}

View File

@ -77,9 +77,9 @@ actual class NewPipeService {
.map { stream ->
org.spotube.plugin_interfaces.plugin_apis.audio.AudioStream.Lossy(
url = stream.content,
codec = stream.codec ?: "unknown",
codec = normalizeCodec(stream.codec ?: "unknown"),
bitrate = stream.bitrate,
container = stream.format?.suffix ?: "unknown")
container = normalizeContainer(stream.format?.suffix ?: "unknown"))
},
videoStreams = listOf(), // Not yet supported
id = id