fix(player): resolve GoogleVideo 403, YouTube video ID extraction, matching relaxation, and radio mix fallback

Co-Authored-By: pgwiz <pgwiz@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [Unreleased]
+
+### Fixes
+- **server/StreamProxy**: Prevent GoogleVideo CDN HTTP 403 Forbidden by injecting `Referer: https://www.youtube.com/` and browser `User-Agent` headers on `googlevideo.com` and `youtube.com` proxy requests, preserving stream range headers (`Range`, `If-Range`) and filtering hop-by-hop headers.
+- **newpipe/RealNewPipeAudioAPI**: Fix YouTube video ID extraction when `track.externalUri` contains YouTube URLs (`v=`, `youtu.be`, `/embed/`, `/shorts/`) instead of passing raw Spotify track IDs to video info lookups.
+- **newpipe/RealNewPipeAudioAPI**: Improve track matching and scoring for Topic channels, VEVO uploads, and official variants with relaxed duration bins, and fall back to top relevance search result when confidence scoring yields no candidates $\ge 0.3$.
+- **newpipe/NewPipeService**: Implement native YouTube radio mix extractor (`RD{videoId}`) across commonMain, androidJvmMain, and iosMain platforms.
+- **audioplayer/DeviceAudioPlayerQueue**: Add native YouTube radio mix fallback for endless playback queue continuation when metadata recommendations fail or are unauthenticated.
+
 ## [6.0.0-beta.0](https://github.com/team-spotube/spotube/compare/v5.1.2...v6.0.0-beta.0) (2026-10-26)
 
 ### Features

composeApp/src/androidJvmMain/kotlin/org/spotube/spotube/core/newpipe/NewPipeService.androidJvm.kt

@@ -155,6 +155,47 @@ actual class NewPipeService {
             )
         }
     }
+
+    actual suspend fun getRadioMix(videoId: String): List<VideoSearchResult> {
+        return withContext(Dispatchers.IO) {
+            val urls = listOf(
+                "https://www.youtube.com/watch?v=$videoId&list=RD$videoId",
+                "https://www.youtube.com/playlist?list=RD$videoId"
+            )
+            for (url in urls) {
+                val items = runCatching {
+                    val extractor = YouTube.getPlaylistExtractor(url)
+                    extractor.fetchPage()
+                    val page = extractor.initialPage
+                    page.items.mapNotNull { item ->
+                        if (item is StreamInfoItem) {
+                            val vId = runCatching { Url(item.url).parameters["v"] }.getOrNull()
+                                ?: item.url.substringAfter("watch?v=", "").substringBefore("&")
+                                .takeIf { it.isNotEmpty() }
+                                ?: item.url.substringAfterLast("/")
+                            VideoSearchResult(
+                                id = vId,
+                                title = item.name,
+                                url = item.url,
+                                uploader = item.uploaderName,
+                                durationMs = secondsToMs(item.duration),
+                                thumbnailUrl = item.thumbnails.firstOrNull()?.url.orEmpty()
+                            )
+                        } else null
+                    }
+                }.onFailure {
+                    logger.w(it) { "Failed to extract YouTube radio mix from $url" }
+                }.getOrNull()
+
+                if (!items.isNullOrEmpty()) {
+                    logger.i { "Extracted ${items.size} tracks from YouTube radio mix for video $videoId" }
+                    return@withContext items
+                }
+            }
+            logger.w { "Unable to fetch radio mix for video $videoId" }
+            emptyList()
+        }
+    }
 }
 
 /**

composeApp/src/commonMain/kotlin/org/spotube/spotube/core/audioplayer/DeviceAudioPlayerQueue.kt

@@ -18,7 +18,12 @@
 package org.spotube.spotube.core.audioplayer
 
 import org.spotube.plugin_interfaces.plugin_apis.audio.StreamProtocol
+import org.spotube.plugin_interfaces.plugin_apis.metadata.artist.MetadataArtist
+import org.spotube.plugin_interfaces.plugin_apis.metadata.common.Thumbnail
+import org.spotube.plugin_interfaces.plugin_apis.metadata.track.MetadataTrack
 import org.spotube.spotube.core.di.injectLogger
+import org.spotube.spotube.core.newpipe.NewPipeService
+import org.spotube.spotube.core.newpipe.SearchKind
 import org.spotube.spotube.modules.blacklist.BlacklistRepository
 import org.spotube.spotube.modules.plugin.PluginProvider
 import org.spotube.spotube.modules.settings.SettingsProvider
@@ -330,19 +335,74 @@ class DeviceAudioPlayerQueue(
         isFetchingRecommendations = true
         logger.i { "Endless playback: fetching recommendations with ${seedTrackIds.size} seed tracks" }
 
-        val metadataService = pluginProvider.selectedMetadataPlugin.value ?: run {
-            logger.w { "Endless playback: no metadata plugin available" }
-            isFetchingRecommendations = false
-            return
-        }
+        val metadataService = pluginProvider.selectedMetadataPlugin.value
+        var recommendations: List<MetadataTrack> = emptyList()
 
-        try {
-            val recommendations = metadataService.use {
+        if (metadataService != null) {
+            recommendations = runCatching {
+                metadataService.use {
                     metadataTrackAPI.recommendationsBasedOnTracks(seedTrackIds, limit = 20)
                 }
+            }.getOrDefault(emptyList())
+        }
 
         if (recommendations.isEmpty()) {
-                logger.w { "Endless playback: no recommendations returned" }
+            logger.i { "Endless playback: metadata recommendations unavailable or empty, falling back to native YouTube radio mix" }
+            val newPipeService = NewPipeService()
+            for (seedTrack in seedTracks) {
+                val seed = seedTrack.track
+                val videoId = if (seed.id.matches(Regex("^[a-zA-Z0-9_-]{11}$"))) {
+                    seed.id
+                } else {
+                    val extracted = seed.externalUri?.let { uri ->
+                        Regex("""[?&]v=([a-zA-Z0-9_-]{11})""").find(uri)?.groupValues?.get(1)
+                            ?: Regex("""youtu\.be/([a-zA-Z0-9_-]{11})""").find(uri)?.groupValues?.get(1)
+                    }
+                    extracted ?: runCatching {
+                        val artist = seed.artists.firstOrNull()?.name.orEmpty()
+                        newPipeService.searchVideos("${seed.title} $artist".trim(), SearchKind.Songs)
+                            .firstOrNull()?.id
+                    }.getOrNull()
+                }
+
+                if (!videoId.isNullOrBlank()) {
+                    val radioMix = runCatching { newPipeService.getRadioMix(videoId) }.getOrDefault(emptyList())
+                    if (radioMix.isNotEmpty()) {
+                        logger.i { "Endless playback: got ${radioMix.size} tracks from YouTube radio mix for video $videoId" }
+                        recommendations = radioMix.map { video ->
+                            MetadataTrack(
+                                id = video.id,
+                                title = video.title,
+                                durationMs = video.durationMs,
+                                trackNumber = null,
+                                discNumber = null,
+                                artists = listOf(
+                                    MetadataArtist.Basic(
+                                        id = video.uploader,
+                                        name = video.uploader,
+                                        thumbnails = emptyList(),
+                                        externalUri = null
+                                    )
+                                ),
+                                album = null,
+                                thumbnails = if (video.thumbnailUrl.isNotBlank()) {
+                                    listOf(Thumbnail(url = video.thumbnailUrl, width = 300, height = 300))
+                                } else null,
+                                explicit = null,
+                                popularity = null,
+                                isrcCode = null,
+                                externalUri = "https://www.youtube.com/watch?v=${video.id}"
+                            )
+                        }
+                        break
+                    }
+                }
+            }
+        }
+
+        try {
+            if (recommendations.isEmpty()) {
+                logger.w { "Endless playback: no recommendations or radio mix tracks found" }
                 isFetchingRecommendations = false
                 return
             }
@@ -369,7 +429,7 @@ class DeviceAudioPlayerQueue(
             logger.i { "Endless playback: adding ${newEntries.size} recommended tracks to queue" }
             addAllToQueue(newEntries)
         } catch (e: Exception) {
-            logger.e(e) { "Endless playback: failed to fetch recommendations" }
+            logger.e(e) { "Endless playback: failed to add recommendations to queue" }
         } finally {
             isFetchingRecommendations = false
         }

composeApp/src/commonMain/kotlin/org/spotube/spotube/core/newpipe/NewPipeService.kt

@@ -28,4 +28,5 @@ enum class SearchKind {
 expect class NewPipeService() {
     suspend fun searchVideos(query: String, kind: SearchKind = SearchKind.Songs): List<VideoSearchResult>
     suspend fun getVideoInfo(id: String): VideoInfo
+    suspend fun getRadioMix(videoId: String): List<VideoSearchResult>
 }

composeApp/src/commonMain/kotlin/org/spotube/spotube/core/server/StreamProxy.kt

@@ -34,6 +34,7 @@ import io.ktor.http.HeadersBuilder
 import io.ktor.http.HttpHeaders
 import io.ktor.http.HttpMethod
 import io.ktor.http.HttpStatusCode
+import io.ktor.http.Url
 import io.ktor.http.content.OutgoingContent
 import io.ktor.http.decodeURLQueryComponent
 import io.ktor.http.encodeURLParameter
@@ -72,6 +73,9 @@ internal class StreamProxy(
 
     companion object {
         private const val HOST = "127.0.0.1"
+        private const val YOUTUBE_REFERER = "https://www.youtube.com/"
+        private const val BROWSER_USER_AGENT =
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
     }
 
     private val fileSystem = FileSystem.SYSTEM
@@ -233,8 +237,7 @@ internal class StreamProxy(
         val response = runCatching {
             httpClient.request(manifestUrl) {
                 method = HttpMethod.Get
-                call.forwardRequestHeaderIfPresent(headers, HttpHeaders.Accept)
-                call.forwardRequestHeaderIfPresent(headers, HttpHeaders.UserAgent)
+                applyUpstreamHeaders(call, this, manifestUrl, includeRange = false)
             }
         }.getOrElse { throwable ->
             logger.w(throwable) { "Failed to fetch manifest for track $trackId" }
@@ -284,9 +287,7 @@ internal class StreamProxy(
         runCatching {
             httpClient.prepareRequest(segmentUrl) {
                 method = HttpMethod.Get
-                call.forwardRequestHeaderIfPresent(headers, HttpHeaders.Range)
-                call.forwardRequestHeaderIfPresent(headers, HttpHeaders.Accept)
-                call.forwardRequestHeaderIfPresent(headers, HttpHeaders.UserAgent)
+                applyUpstreamHeaders(call, this, segmentUrl, includeRange = true)
             }.execute { response -> respondWithSegment(call, response, trackId) }
         }.onFailure { throwable ->
             if (throwable is CancellationException) throw throwable
@@ -343,10 +344,7 @@ internal class StreamProxy(
             val failure = runCatching {
                 httpClient.prepareRequest(currentUrl) {
                     method = requestMethod
-                    call.forwardRequestHeaderIfPresent(headers, HttpHeaders.Range)
-                    call.forwardRequestHeaderIfPresent(headers, HttpHeaders.IfRange)
-                    call.forwardRequestHeaderIfPresent(headers, HttpHeaders.Accept)
-                    call.forwardRequestHeaderIfPresent(headers, HttpHeaders.UserAgent)
+                    applyUpstreamHeaders(call, this, currentUrl, includeRange = true)
                 }.execute { response ->
                     logger.i {
                         "[Req] upstream headers for $trackId after ${upstreamStart.elapsedNow()}: " +
@@ -578,6 +576,35 @@ internal class StreamProxy(
     private fun encodeUrl(url: String): String = url.encodeURLParameter()
     private fun decodeUrl(encoded: String): String = encoded.decodeURLQueryComponent()
 
+    private fun isYouTubeOrGoogleVideo(url: String): Boolean {
+        val host = runCatching { Url(url).host.lowercase() }.getOrNull() ?: ""
+        return host.endsWith("googlevideo.com") ||
+            host.endsWith("youtube.com") ||
+            url.contains("googlevideo.com", ignoreCase = true) ||
+            url.contains("youtube.com", ignoreCase = true)
+    }
+
+    private fun applyUpstreamHeaders(
+        call: ApplicationCall,
+        builder: io.ktor.client.request.HttpRequestBuilder,
+        targetUrl: String,
+        includeRange: Boolean
+    ) {
+        if (includeRange) {
+            call.forwardRequestHeaderIfPresent(builder.headers, HttpHeaders.Range)
+            call.forwardRequestHeaderIfPresent(builder.headers, HttpHeaders.IfRange)
+        }
+        call.forwardRequestHeaderIfPresent(builder.headers, HttpHeaders.Accept)
+
+        if (isYouTubeOrGoogleVideo(targetUrl)) {
+            builder.headers[HttpHeaders.Referrer] = YOUTUBE_REFERER
+            builder.headers[HttpHeaders.UserAgent] = BROWSER_USER_AGENT
+        } else {
+            call.forwardRequestHeaderIfPresent(builder.headers, HttpHeaders.UserAgent)
+            call.forwardRequestHeaderIfPresent(builder.headers, HttpHeaders.Referrer)
+        }
+    }
+
     private fun ApplicationCall.forwardRequestHeaderIfPresent(
         builder: HeadersBuilder,
         headerName: String
@@ -590,8 +617,12 @@ internal class StreamProxy(
     private fun upstreamHeadersToForward(upstream: HttpResponse): Headers {
         return HeadersBuilder().apply {
             upstream.headers.forEach { name, values ->
+                if (!name.equals(HttpHeaders.TransferEncoding, ignoreCase = true) &&
+                    !name.equals(HttpHeaders.Connection, ignoreCase = true)
+                ) {
                     values.forEach { value -> append(name, value) }
                 }
+            }
         }.build()
     }
 }

composeApp/src/commonMain/kotlin/org/spotube/spotube/core/zipline/plugin_apis/newpipe_yt/RealNewPipeAudioAPI.kt

@@ -55,29 +55,47 @@ class RealNewPipeAudioAPI : AudioAPI, KoinComponent {
         var confidence = 0f
 
         val videoTitle = normalize(video.title)
-        val isTopicChannel = video.uploader.trim().endsWith("- Topic", ignoreCase = true)
-        val uploader = normalize(video.uploader.replace(Regex("(?i)\\s*-\\s*topic$"), ""))
+        val isTopicChannel = video.uploader.trim().endsWith("- Topic", ignoreCase = true) ||
+            video.uploader.trim().contains("Topic", ignoreCase = true)
+        val uploaderRaw = video.uploader.replace(Regex("(?i)\\s*-\\s*topic$"), "").trim()
+        val uploader = normalize(uploaderRaw)
         val artistNames = track.artists.map { normalize(it.name) }.filter { it.isNotEmpty() }
         val title = coreTitle(track.title)
 
         val videoTokens = videoTitle.split(' ').toSet()
         val isTitleMatch = title.isNotEmpty() &&
-            (title in videoTitle || title.split(' ').all { it in videoTokens })
+            (title in videoTitle || videoTitle in title || title.split(' ').filter { it.length > 2 }.all { it in videoTokens })
         val isArtistInTitle = artistNames.any { it in videoTitle }
         val isAlbumInTitle = track.album?.title?.let { normalize(it) }
             ?.takeIf { it.isNotEmpty() }?.let { it in videoTitle } ?: false
-        val isUploaderMatch = uploader.isNotEmpty() && artistNames.any { it in uploader || uploader in it }
-        // Unknown duration (0) must neither reward nor punish
-        val durationDiff = if (video.durationMs > 0) kotlin.math.abs(video.durationMs - track.durationMs) else Long.MAX_VALUE
+
+        // Normalize whitespace and punctuation for uploader comparison (e.g. "Taylor Swift" vs "TaylorSwiftVEVO")
+        val uploaderCompact = uploader.replace(Regex("(?i)vevo$"), "").replace(" ", "")
+        val artistCompacts = artistNames.map { it.replace(" ", "") }
+        val isUploaderMatch = uploader.isNotEmpty() && (
+            artistNames.any { it in uploader || uploader in it } ||
+            (uploaderCompact.isNotEmpty() && artistCompacts.any { it.isNotEmpty() && (it in uploaderCompact || uploaderCompact in it) })
+        )
+        val isVevo = video.uploader.contains("VEVO", ignoreCase = true)
+        val isOfficialTrack = listOf("official audio", "official video", "official music video", "official visualizer", "official lyric video").any { videoTitle.contains(it) }
+
+        val hasValidDurations = video.durationMs > 0 && track.durationMs > 0
+        val durationDiff = if (hasValidDurations) kotlin.math.abs(video.durationMs - track.durationMs) else Long.MAX_VALUE
 
         if (isTitleMatch) confidence += 0.3f
         if (isUploaderMatch) confidence += 0.3f
+        if (isVevo) confidence += 0.15f
+        if (isOfficialTrack) confidence += 0.1f
+        if (isTopicChannel) confidence += if (isUploaderMatch) 0.2f else 0.1f
+
         if (durationDiff <= 10_000) confidence += 0.2f
-        else if (durationDiff <= 30_000) confidence += 0.1f
+        else if (durationDiff <= 30_000) confidence += 0.15f
+        else if (durationDiff <= 60_000) confidence += 0.1f
+        else if (durationDiff <= 120_000) confidence += 0.05f
+        else if (!hasValidDurations) confidence += 0.1f // Don't penalize when duration is unknown
+
         if (isArtistInTitle) confidence += 0.1f
         if (isAlbumInTitle) confidence += 0.1f
-        // Auto-generated "Artist - Topic" uploads are the studio audio, preferred over videos
-        if (isTopicChannel && isUploaderMatch) confidence += 0.1f
         confidence = confidence.coerceAtMost(1f)
 
         logger.i {
@@ -122,12 +140,29 @@ class RealNewPipeAudioAPI : AudioAPI, KoinComponent {
         )
     )
 
+    private fun extractYouTubeVideoId(urlOrId: String?): String? {
+        if (urlOrId.isNullOrBlank()) return null
+        val trimmed = urlOrId.trim()
+        if (trimmed.matches(youtubeIDRegex)) return trimmed
+
+        val vParamRegex = Regex("""[?&]v=([a-zA-Z0-9_-]{11})(?:[&#]|$)""")
+        vParamRegex.find(trimmed)?.groupValues?.get(1)?.let { return it }
+
+        val youtuBeRegex = Regex("""youtu\.be/([a-zA-Z0-9_-]{11})(?:[?#]|$)""")
+        youtuBeRegex.find(trimmed)?.groupValues?.get(1)?.let { return it }
+
+        val pathRegex = Regex("""youtube\.com/(?:embed|v|shorts)/([a-zA-Z0-9_-]{11})(?:[?#]|$)""")
+        pathRegex.find(trimmed)?.groupValues?.get(1)?.let { return it }
+
+        return null
+    }
+
     override suspend fun getStreamsByTrack(track: MetadataTrack): List<AudioSource> {
-        val isYouTubeUrl =
-            track.externalUri != null && (track.externalUri!!.contains("youtube.com") || track.externalUri!!.contains(
-                "youtu.be"
-            ))
-        val isYouTubeID = track.id.matches(youtubeIDRegex)
+        val resolvedVideoId = if (track.id.matches(youtubeIDRegex)) {
+            track.id
+        } else {
+            extractYouTubeVideoId(track.externalUri)
+        }
 
         logger.i {
             "Searching streams for track(${track.id}): ${track.title} by ${
@@ -136,9 +171,9 @@ class RealNewPipeAudioAPI : AudioAPI, KoinComponent {
                 ) { it.name }
             }"
         }
-        if (isYouTubeID || isYouTubeUrl) {
-            logger.i { "Track has YouTube ID or URL, fetching video info directly" }
-            return newPipeService.getVideoInfo(track.id).let { videoInfo ->
+        if (resolvedVideoId != null) {
+            logger.i { "Track has YouTube ID or URL ($resolvedVideoId), fetching video info directly" }
+            return newPipeService.getVideoInfo(resolvedVideoId).let { videoInfo ->
                 listOf(
                     AudioSource.Streamed(
                         id = videoInfo.id,
@@ -228,19 +263,33 @@ class RealNewPipeAudioAPI : AudioAPI, KoinComponent {
 
         if (matched.isNotEmpty()) return matched.take(5)
 
-        // Nothing scored well. Playing YouTube's best text-search guess beats failing playback,
-        // and ISRC results are skipped because an ISRC text search tends to return unrelated songs.
-        val fallback = textSearchResults.values.toList()
+        // When confidence scoring yields no candidates >= 0.3, ensure the top relevance search
+        // result is returned as fallback rather than throwing TrackNotFoundError or failing.
+        val fallback = if (textSearchResults.isNotEmpty()) {
+            textSearchResults.values.toList()
                 .sortedByDescending { it.confidence } // stable: ties keep YouTube's relevance order
                 .take(5)
-            .ifEmpty { ranked.take(5) }
+        } else if (ranked.isNotEmpty()) {
+            ranked.take(5)
+        } else {
+            runCatching {
+                newPipeService.searchVideos(plainTitle.ifEmpty { track.title }, SearchKind.Videos)
+                    .map { transformVideo(track, it) }
+            }.getOrDefault(emptyList()).take(5)
+        }
+
+        if (fallback.isNotEmpty()) {
             logger.w {
-            "[Search] no confident match for '${track.title}'; falling back to best guesses: " +
+                "[Search] no confident match for '${track.title}'; falling back to top relevance result: " +
                     fallback.joinToString { "${it.id} conf=${it.confidence} '${it.title}'" }
             }
             return fallback
         }
 
+        logger.w { "[Search] no search results found at all for '${track.title}'" }
+        return emptyList()
+    }
+
     override suspend fun getStreamsOfAudioSource(source: AudioSource.Basic): List<AudioSource.Streamed> {
         return newPipeService.getVideoInfo(source.id).let { videoInfo ->
             listOf(

composeApp/src/iosMain/kotlin/org/spotube/spotube/core/newpipe/NewPipeService.ios.kt

@@ -86,4 +86,29 @@ actual class NewPipeService {
             )
         }
     }
+
+    actual suspend fun getRadioMix(videoId: String): List<VideoSearchResult> {
+        return withContext(Dispatchers.IO) {
+            runCatching {
+                val extractor = YouTube.getPlaylistExtractor("https://www.youtube.com/watch?v=$videoId&list=RD$videoId")
+                extractor.fetchPage()
+                val results = extractor.initialPage
+                results.items.mapNotNull { item ->
+                    if (item is StreamInfoItem) {
+                        VideoSearchResult(
+                            title = item.name,
+                            url = item.url,
+                            uploader = item.uploaderName ?: "Unknown",
+                            durationMs = secondsToMs(item.duration),
+                            thumbnailUrl = item.thumbnails.first().url,
+                            id = Url(item.url).parameters["v"]
+                                ?: item.url.substringAfter("watch?v=", "").substringBefore("&")
+                        )
+                    } else {
+                        null
+                    }
+                }
+            }.getOrDefault(emptyList())
+        }
+    }
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.0-all.zip
-networkTimeout=10000
+networkTimeout=600000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists