Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Advanced filtering support #387

Merged
merged 4 commits into from
Nov 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 69 additions & 9 deletions src/main/kotlin/org/koitharu/kotatsu/parsers/MangaParser.kt
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,27 @@ abstract class MangaParser @InternalParsersApi constructor(
*
* For better performance use [EnumSet] for more than one item.
*/
abstract val sortOrders: Set<SortOrder>
abstract val availableSortOrders: Set<SortOrder>

/**
* Supported [MangaState] variants for filtering. May be empty.
*
* For better performance use [EnumSet] for more than one item.
*/
open val availableStates: Set<MangaState>
get() = emptySet()

/**
* Whether parser supports filtering by more than one tag
*/
open val isMultipleTagsSupported: Boolean = true

@Deprecated(
message = "Use availableSortOrders instead",
replaceWith = ReplaceWith("availableSortOrders"),
)
val sortOrders: Set<SortOrder>
get() = availableSortOrders

val config by lazy { context.getConfig(source) }

Expand All @@ -49,7 +69,7 @@ abstract class MangaParser @InternalParsersApi constructor(
*/
protected open val defaultSortOrder: SortOrder
get() {
val supported = sortOrders
val supported = availableSortOrders
return SortOrder.entries.first { it in supported }
}

Expand All @@ -62,11 +82,15 @@ abstract class MangaParser @InternalParsersApi constructor(
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param query search query, may be null or empty if no search needed
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
*/
@JvmSynthetic
@InternalParsersApi
@Deprecated(
"Use getList with filter instead",
replaceWith = ReplaceWith("getList(offset, filter)"),
)
abstract suspend fun getList(
offset: Int,
query: String?,
Expand All @@ -80,20 +104,45 @@ abstract class MangaParser @InternalParsersApi constructor(
* @param offset starting from 0 and used for pagination.
* @param query search query
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Search(query))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(offset: Int, query: String): List<Manga> {
return getList(offset, query, null, defaultSortOrder)
return getList(offset, MangaListFilter.Search(query))
}

/**
* Parse list of manga by specified criteria
*
* @param offset starting from 0 and used for pagination.
* Note than passed value may not be divisible by internal page size, so you should adjust it manually.
* @param tags genres for filtering, values from [getTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [sortOrders] or null for default value
* @param tags genres for filtering, values from [getAvailableTags] and [Manga.tags]. May be null or empty
* @param sortOrder one of [availableSortOrders] or null for default value
*/
@Deprecated(
"Use getList with filter instead",
ReplaceWith(
"getList(offset, MangaListFilter.Advanced(sortOrder, tags, null, emptySet()))",
"org.koitharu.kotatsu.parsers.model.MangaListFilter",
),
)
open suspend fun getList(offset: Int, tags: Set<MangaTag>?, sortOrder: SortOrder?): List<Manga> {
return getList(offset, null, tags, sortOrder ?: defaultSortOrder)
return getList(
offset,
MangaListFilter.Advanced(sortOrder ?: defaultSortOrder, tags.orEmpty(), null, emptySet()),
)
}

open suspend fun getList(offset: Int, filter: MangaListFilter?): List<Manga> {
return when (filter) {
is MangaListFilter.Advanced -> getList(offset, null, filter.tags, filter.sortOrder)
is MangaListFilter.Search -> getList(offset, filter.query, null, defaultSortOrder)
null -> getList(offset, null, null, defaultSortOrder)
}
}

/**
Expand All @@ -117,7 +166,18 @@ abstract class MangaParser @InternalParsersApi constructor(
/**
* Fetch available tags (genres) for source
*/
abstract suspend fun getTags(): Set<MangaTag>
abstract suspend fun getAvailableTags(): Set<MangaTag>

/**
* Fetch available locales for multilingual sources
*/
open suspend fun getAvailableLocales(): Set<Locale> = emptySet()

@Deprecated(
message = "Use getAvailableTags instead",
replaceWith = ReplaceWith("getAvailableTags()"),
)
suspend fun getTags(): Set<MangaTag> = getAvailableTags()

/**
* Parse favicons from the main page of the source`s website
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package org.koitharu.kotatsu.parsers.model

import java.util.*

sealed interface MangaListFilter {

fun isEmpty(): Boolean

val sortOrder: SortOrder?

data class Search(
@JvmField val query: String,
) : MangaListFilter {

override val sortOrder: SortOrder? = null

override fun isEmpty() = query.isBlank()
}

data class Advanced(
override val sortOrder: SortOrder,
@JvmField val tags: Set<MangaTag>,
@JvmField val locale: Locale?,
@JvmField val states: Set<MangaState>,
) : MangaListFilter {

override fun isEmpty(): Boolean = tags.isEmpty() && locale == null && states.isEmpty()
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
package org.koitharu.kotatsu.parsers.model

enum class MangaState {
ONGOING, FINISHED, ABANDONED
ONGOING, FINISHED, ABANDONED, PAUSED
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
searchPageSize = 20,
) {

override val sortOrders: Set<SortOrder> = EnumSet.of(
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.NEWEST,
SortOrder.UPDATED,
SortOrder.POPULARITY,
Expand Down Expand Up @@ -159,7 +159,7 @@ internal class BatoToParser(context: MangaLoaderContext) : PagedMangaParser(
throw ParseException("Cannot find images list", fullUrl)
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val scripts = webClient.httpGet(
"https://${domain}/browse",
).parseHtml().selectOrThrow("script")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex

override val configKeyDomain = ConfigKey.Domain("comick.app")

override val sortOrders: Set<SortOrder> = EnumSet.of(
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.UPDATED,
SortOrder.RATING,
Expand Down Expand Up @@ -137,7 +137,7 @@ internal class ComickFunParser(context: MangaLoaderContext) : MangaParser(contex
}
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val sparseArray = cachedTags ?: loadTags()
val set = ArraySet<MangaTag>(sparseArray.size())
for (i in 0 until sparseArray.size()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ internal class ExHentaiParser(
context: MangaLoaderContext,
) : PagedMangaParser(context, MangaSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider {

override val sortOrders: Set<SortOrder> = Collections.singleton(
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.NEWEST,
)

Expand Down Expand Up @@ -213,7 +213,7 @@ internal class ExHentaiParser(
return doc.body().requireElementById("img").attrAsAbsoluteUrl("src")
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://${domain}").parseHtml()
val root = doc.body().requireElementById("searchbox").selectFirstOrThrow("table")
return root.select("div.cs").mapNotNullToSet { div ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@ import java.util.*
internal class ImHentai(context: MangaLoaderContext) :
PagedMangaParser(context, MangaSource.IMHENTAI, pageSize = 20) {

override val sortOrders: Set<SortOrder> =
override val availableSortOrders: Set<SortOrder> =
EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING)

override val configKeyDomain = ConfigKey.Domain("imhentai.xxx")

override val isMultipleTagsSupported = false

override suspend fun getListPage(
page: Int,
query: String?,
Expand Down Expand Up @@ -83,7 +85,7 @@ internal class ImHentai(context: MangaLoaderContext) :

//Tags are deliberately reduced because there are too many and this slows down the application.
//only the most popular ones are taken.
override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
return coroutineScope {
(1..3).map { page ->
async { getTags(page) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ internal abstract class LineWebtoonsParser(
source: MangaSource,
) : MangaParser(context, source) {

override val isMultipleTagsSupported = false

private val signer by lazy {
WebtoonsUrlSigner("gUtPzJFZch4ZyAGviiyH94P99lQ3pFdRTwpJWDlSGFfwgpr6ses5ALOxWHOIT7R1")
}
Expand All @@ -39,7 +41,7 @@ internal abstract class LineWebtoonsParser(
private val apiDomain = "global.apis.naver.com"
private val staticDomain = "webtoon-phinf.pstatic.net"

override val sortOrders: Set<SortOrder> = EnumSet.of(
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
// doesn't actually sort by rating, but by likes
// this should be fine though
Expand Down Expand Up @@ -235,7 +237,7 @@ internal abstract class LineWebtoonsParser(
)
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
return makeRequest("/lineWebtoon/webtoon/challengeGenreList.json")
.getJSONObject("genreList")
.getJSONArray("challengeGenres")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context

override val configKeyDomain = ConfigKey.Domain("mangadex.org")

override val sortOrders: EnumSet<SortOrder> = EnumSet.of(
override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.ALPHABETICAL,
SortOrder.NEWEST,
Expand Down Expand Up @@ -159,7 +159,7 @@ internal class MangaDexParser(context: MangaLoaderContext) : MangaParser(context
}
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson()
.getJSONArray("data")
return tags.mapJSONToSet { jo ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ internal abstract class NineMangaParser(
.add("Accept-Language", "en-US;q=0.7,en;q=0.3")
.build()

override val sortOrders: Set<SortOrder> = Collections.singleton(
override val availableSortOrders: Set<SortOrder> = Collections.singleton(
SortOrder.POPULARITY,
)

Expand Down Expand Up @@ -158,7 +158,7 @@ internal abstract class NineMangaParser(
private var tagCache: ArrayMap<String, MangaTag>? = null
private val mutex = Mutex()

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
return getOrCreateTagMap().values.toSet()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ internal abstract class AnimeBootstrapParser(

override val configKeyDomain = ConfigKey.Domain(domain)

override val sortOrders: Set<SortOrder> = EnumSet.of(
override val isMultipleTagsSupported = false

override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.UPDATED,
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
Expand Down Expand Up @@ -93,7 +95,7 @@ internal abstract class AnimeBootstrapParser(
}
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("div.product__page__filter div:contains(Genre:) option ").mapNotNullToSet { option ->
val key = option.attr("value") ?: return@mapNotNullToSet null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,16 @@ internal class PapScan(context: MangaLoaderContext) :

override val sourceLocale: Locale = Locale.ENGLISH

override val isMultipleTagsSupported = false

override val listUrl = "/liste-manga"

override val selectState = "div.anime__details__widget li:contains(En cours)"
override val selectTag = "div.anime__details__widget li:contains(Genre) a"

override val selectChapter = "ul.chapters li"

override val sortOrders: Set<SortOrder> = EnumSet.of(
override val availableSortOrders: Set<SortOrder> = EnumSet.of(
SortOrder.POPULARITY,
SortOrder.ALPHABETICAL,
)
Expand Down Expand Up @@ -85,7 +87,7 @@ internal class PapScan(context: MangaLoaderContext) :
}
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain$listUrl").parseHtml()
return doc.select("a.category ").mapNotNullToSet { a ->
val key = a.attr("href").substringAfterLast('=')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import java.util.*
@MangaSourceParser("FLIXSCANS", "FlixScans", "ar")
internal class FlixScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.FLIXSCANS, 18) {

override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED)
override val configKeyDomain = ConfigKey.Domain("flixscans.com")

override suspend fun getListPage(
Expand Down Expand Up @@ -70,7 +70,7 @@ internal class FlixScans(context: MangaLoaderContext) : PagedMangaParser(context
}
}

override suspend fun getTags(): Set<MangaTag> {
override suspend fun getAvailableTags(): Set<MangaTag> {
val doc = webClient.httpGet("https://$domain/search/advance").parseHtml()
val json = JSONArray(doc.requireElementById("__NUXT_DATA__").data())
val tagsList = json.getJSONArray(3).toString().replace("[", "").replace("]", "").split(",")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ import java.util.*
@MangaSourceParser("MANGASTORM", "MangaStorm", "ar")
internal class MangaStorm(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.MANGASTORM, 30) {

override val sortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY)
override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY)
override val configKeyDomain = ConfigKey.Domain("mangastorm.org")
override val isMultipleTagsSupported = false

override val headers: Headers = Headers.Builder()
.add("User-Agent", UserAgents.CHROME_DESKTOP)
Expand Down Expand Up @@ -69,7 +70,7 @@ internal class MangaStorm(context: MangaLoaderContext) : PagedMangaParser(contex
}
}

override suspend fun getTags(): Set<MangaTag> = emptySet()
override suspend fun getAvailableTags(): Set<MangaTag> = emptySet()

override suspend fun getDetails(manga: Manga): Manga {
val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml()
Expand Down
Loading