kitty/spotube
1
0
Fork 0
mirror of https://github.com/KRTirtho/spotube.git synced 2026-11-04 10:13:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(user-agents): implement dynamic user agent fetching and caching mechanism

Browse Source
This commit is contained in:
Kingkor Roy Tirtho 2026-10-16 13:47:08 +06:00
parent 83115ea99a
commit ba62ff03c3
1 changed files with 180 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
composeApp/src/commonMain/kotlin/dev/krtirtho/spotube/core/tools/user_agents/UserAgents.kt
View File

@ -17,18 +17,185 @@
package dev.krtirtho.spotube.core.tools.user_agents
object UserAgents {
private val agents = listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"Mozilla/5.0 (Linux; Android 15; ALI-NX1 Build/HONORALI-N21) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7727.92 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 14; SM-A528B Build/UP1A.231005.007) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7727.108 Mobile Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:68.0) Gecko/20100101 Firefox/68.0 Waterfox/56.6.2021.2816",
"Mozilla/5.0 (Macintosh; Intel Mac OS X10.14; rv:65.0) Gecko/20100101 Firefox/65.0 Waterfox/56.2.7",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Vivaldi/7.6.3797.63",
"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.91 Safari/537.36 Vivaldi/1.92.917.39",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Vivaldi/7.5.3735.47"
)
import co.touchlab.kermit.Logger
import dev.krtirtho.spotube.PlatformType
import dev.krtirtho.spotube.core.paths.Paths
import dev.krtirtho.spotube.getPlatform
import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
import io.ktor.client.request.get
import io.ktor.client.statement.bodyAsText
import io.ktor.http.isSuccess
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.serialization.Serializable
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.json.Json
import okio.FileSystem
import okio.Path.Companion.toPath
import okio.SYSTEM
import kotlin.time.Clock
fun random(): String = agents.random()
/**
* Provides browser User-Agent strings that match the platform the app is running on
* (Android UAs only on Android, iOS only on iOS, and desktop UAs only on the matching
* Windows / macOS / Linux host).
*
* The primary source is the `top-user-agents` list maintained by Microlink (the 100 most
* used UAs, kept current), cached on disk for [CACHE_TTL_MS]. The hardcoded [FALLBACKS]
* are used whenever nothing has been fetched yet or the fetch fails.
*
* [random] is intentionally synchronous and never blocks on the network: it serves
* whatever is cached (even stale) and refreshes in the background when needed.
*/
object UserAgents {
private const val SOURCE_URL =
"https://cdn.jsdelivr.net/npm/top-user-agents@latest/src/index.json"
private const val CACHE_FILE_NAME = "user_agents_cache.json"
private const val CACHE_TTL_MS = 24L * 60 * 60 * 1000 // 1 day
private val logger = Logger.withTag("UserAgents")
private val json = Json { ignoreUnknownKeys = true }
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val refreshMutex = Mutex()
private val platformType: PlatformType = getPlatform().type
@Serializable
private data class CacheEntry(val fetchedAtMs: Long, val agents: List<String>)
private var remoteAgents: List<String> = emptyList()
private var fetchedAtMs: Long = 0
// Loads the disk cache once and kicks off a background refresh if it is missing/stale.
private val bootstrap by lazy {
loadFromDisk()
if (isStale()) scope.launch { refresh() }
}
fun random(): String {
bootstrap
return pool().random()
}
/**
* Fetches the latest list from the network and caches it. No-op while the cache is
* still fresh unless [force] is set. Failures keep whatever data is already cached.
*/
suspend fun refresh(force: Boolean = false) = refreshMutex.withLock {
if (!force && !isStale()) return@withLock
try {
val agents = fetchRemote().filter(::isValidAgent)
if (agents.isEmpty()) return@withLock
remoteAgents = agents
fetchedAtMs = now()
saveToDisk(CacheEntry(fetchedAtMs, agents))
} catch (e: Exception) {
logger.w(e) { "Failed to refresh user agents, using cached/fallback ones" }
}
}
private fun pool(): List<String> {
val matching = remoteAgents.filter { matchesPlatform(it, platformType) }
return matching.ifEmpty { FALLBACKS.getValue(platformType) }
}
private fun isStale() = now() - fetchedAtMs >= CACHE_TTL_MS
private fun now() = Clock.System.now().toEpochMilliseconds()
private fun isValidAgent(ua: String) = ua.startsWith("Mozilla/")
private suspend fun fetchRemote(): List<String> {
val client = HttpClient(CIO)
try {
val res = client.get(SOURCE_URL)
if (!res.status.isSuccess()) error("Unexpected status ${res.status}")
return json.decodeFromString(ListSerializer(String.serializer()), res.bodyAsText())
} finally {
client.close()
}
}
private fun cachePath() = (Paths().getApplicationCacheDirPath() + "/" + CACHE_FILE_NAME).toPath()
private fun loadFromDisk() {
try {
val path = cachePath()
val fs = FileSystem.SYSTEM
if (!fs.exists(path)) return
val entry = json.decodeFromString<CacheEntry>(fs.read(path) { readUtf8() })
val agents = entry.agents.filter(::isValidAgent)
if (agents.isEmpty()) return
remoteAgents = agents
fetchedAtMs = entry.fetchedAtMs
} catch (e: Exception) {
logger.w(e) { "Failed to read user agents cache" }
}
}
private fun saveToDisk(entry: CacheEntry) {
try {
val path = cachePath()
val tmp = "$path.tmp".toPath()
val fs = FileSystem.SYSTEM
path.parent?.let { fs.createDirectories(it) }
fs.write(tmp) { writeUtf8(json.encodeToString(CacheEntry.serializer(), entry)) }
fs.atomicMove(tmp, path)
} catch (e: Exception) {
logger.w(e) { "Failed to write user agents cache" }
}
}
internal fun matchesPlatform(ua: String, type: PlatformType): Boolean {
val isIos = "iPhone" in ua || "iPad" in ua || "iPod" in ua
val isAndroid = "Android" in ua
return when (type) {
PlatformType.IOS -> isIos
PlatformType.Android -> isAndroid
PlatformType.Windows -> "Windows NT" in ua && !isAndroid
PlatformType.MacOS -> "Macintosh" in ua && !isIos
PlatformType.Linux -> ("X11" in ua || "Linux" in ua) && !isAndroid && "CrOS" !in ua
// Unknown host: any desktop UA is a safe choice.
PlatformType.Unknown -> !isIos && !isAndroid
}
}
private val FALLBACKS: Map<PlatformType, List<String>> = mapOf(
PlatformType.Windows to listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Vivaldi/7.6.3797.63",
),
PlatformType.MacOS to listOf(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Vivaldi/7.5.3735.47",
),
PlatformType.Linux to listOf(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
),
PlatformType.Android to listOf(
"Mozilla/5.0 (Linux; Android 15; ALI-NX1 Build/HONORALI-N21) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7727.92 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 14; SM-A528B Build/UP1A.231005.007) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/147.0.7727.108 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Mobile Safari/537.36",
),
PlatformType.IOS to listOf(
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/146.0.0.0 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPad; CPU OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1",
),
PlatformType.Unknown to listOf(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
),
)
}