From e992566899d0572ef478a6242af3b3b190fddfa7 Mon Sep 17 00:00:00 2001 From: AwkwardPeak7 Date: Tue, 12 Dec 2023 13:27:46 +0500 Subject: [PATCH 1/4] 999Hentai --- .../site/all/NineNineNineHentaiParser.kt | 401 ++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt new file mode 100644 index 000000000..0a10016f5 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt @@ -0,0 +1,401 @@ +package org.koitharu.kotatsu.parsers.site.all + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.PagedMangaParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.domain +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet +import org.koitharu.kotatsu.parsers.util.json.toJSONList +import org.koitharu.kotatsu.parsers.util.mapToSet +import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("NINENINENINEHENTAI", "999Hentai", type = ContentType.HENTAI) +internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaSource.NINENINENINEHENTAI, size), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("999hentai.net") + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST + ) + + override val isMultipleTagsSupported = false + + override suspend fun getAvailableLocales() = setOf( + Locale.ENGLISH, + Locale.CHINESE, + Locale.JAPANESE, + Locale("es") + ) + + private fun Locale?.getSiteLang(): String { + if (this == null) return "all" + + return when { + equals(Locale.ENGLISH) -> "en" + equals(Locale.CHINESE) -> "cn" + equals(Locale.JAPANESE) -> "jp" + equals(Locale("es")) -> "es" + else -> "all" + } + } + + // Need for disable encoding (with encoding not working) + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.header("Content-Encoding") != null) { + request.newBuilder().removeHeader("Content-Encoding").build() + } else { + request + } + return chain.proceed(newRequest) + } + + private var cdnUrl: String? = null + + private suspend fun getCdnUrl(): String { + if (cdnUrl.isNullOrEmpty()) { + val url = "https://$domain/manga-home" + val response = webClient.httpGet(url).parseHtml() + val cdn = response.selectFirst("img.v-thumbnail")?.attr("data-src") + cdnUrl = cdn?.toHttpUrlOrNull()?.host + } + + return cdnUrl ?: "edge.fast4speed.rsvp" + } + + override suspend fun getAvailableTags(): Set { + val query = """ + queryTags( + search: {format:"tagchapter",sortBy:Popular} + page: 1 + limit: 100 + ) { + edges { + name + } + } + """.trimIndent() + + val tags = apiCall(query) + .getJSONObject("queryTags") + .getJSONArray("edges") + + return tags.mapJSONToSet { + val name = it.getString("name") + MangaTag( + title = name.capitalize(), + key = name, + source = source + ) + } + } + + override suspend fun getListPage(page: Int, filter: MangaListFilter?): List { + return when (filter) { + is MangaListFilter.Advanced -> { + if (filter.tags.isEmpty() && filter.sortOrder == SortOrder.POPULARITY) { + getPopularList(page, filter.locale) + } else { + getSearchList(page, null, filter.locale, filter.tags, filter.sortOrder) + } + } + is MangaListFilter.Search -> { + getSearchList(page, filter.query, null, null, filter.sortOrder) + } + else -> emptyList() + } + } + + private suspend fun getPopularList( + page: Int, + locale: Locale? + ): List { + val query = """ + queryPopularChapters( + size: $size + language: "${locale.getSiteLang()}" + dateRange: 1 + page: $page + ) { + edges { + _id + name + firstPics + } + } + """.trimIndent() + + return apiCall(query) + .getJSONObject("queryPopularChapters") + .getJSONArray("edges") + .toMangaList() + } + + private suspend fun getSearchList( + page: Int, + search: String?, + locale: Locale?, + tags: Set?, + sort: SortOrder?, + ): List { + val searchPayload = buildString { + if (!search.isNullOrEmpty()) { + append("query:\"$search\",") + } + append("language:\"${locale.getSiteLang()}\"") + if (sort == SortOrder.POPULARITY) { + append(",sortBy:Popular") + } + if (!tags.isNullOrEmpty()) { + val tag = tags.oneOrThrowIfMany()!!.key + append(",tags:[\"$tag\"]") + } + } + val query = """ + queryChapters( + limit: $size + search: {$searchPayload} + page: $page + ) { + edges { + _id + name + firstPics + } + } + """.trimIndent() + + return apiCall(query) + .getJSONObject("queryChapters") + .getJSONArray("edges") + .toMangaList() + } + + private suspend fun JSONArray.toMangaList(): List = mapJSON { entry -> + val id = entry.getString("_id") + val name = entry.getString("name") + val cover = runCatching { + entry.getJSONArray("firstPics") + .getJSONObject(0) + .getString("url") + }.getOrNull() + + Manga( + id = generateUid(id), + title = name.replace(shortenTitleRegex, "").trim(), + altTitle = name, + coverUrl = when { + cover?.startsWith("http") == true -> cover + cover == null -> "" + else -> "https://${getCdnUrl()}/$cover" + }, + author = null, + isNsfw = true, + url = id, + publicUrl = "/hchapter/$id".toAbsoluteUrl(domain), + tags = emptySet(), + source = source, + state = MangaState.FINISHED, + rating = RATING_UNKNOWN, + ) + } + + + override suspend fun getDetails(manga: Manga): Manga { + val query = """ + queryChapter( + chapterId: "${manga.url}" + ) { + _id + name + uploadDate + format + description + language + pages + firstPics + tags + } + """.trimIndent() + + val entry = apiCall(query) + .getJSONObject("queryChapter") + + val id = entry.getString("_id") + val name = entry.getString("name") + val cover = runCatching { + entry.getJSONArray("firstPics") + .getJSONObject(0) + .getString("url") + }.getOrNull() + val tags = entry.optJSONArray("tags")?.mapJSON { + SiteTag( + name = it.getString("tagName"), + type = it.getStringOrNull("tagType") + ) + } + return manga.copy( + title = name.replace(shortenTitleRegex, "").trim(), + altTitle = name, + coverUrl = when { + cover?.startsWith("http") == true -> cover + cover == null -> "" + else -> "https://${getCdnUrl()}/$cover" + }, + author = tags?.filter { it.type == "artist" }?.joinToString { it.name.capitalize() }, + isNsfw = true, + tags = tags?.mapToSet { + MangaTag( + title = it.name.capitalize(), + key = it.name, + source = source + ) + }.orEmpty(), + state = MangaState.FINISHED, + description = entry.getStringOrNull("description"), + chapters = listOf( + MangaChapter( + id = generateUid(id), + name = name, + number = 1, + url = id, + uploadDate = kotlin.runCatching { + dateFormat.parse(entry.getString("uploadDate"))!!.time + }.getOrDefault(0L), + branch = when (entry.getStringOrNull("language")) { + "en" -> "English" + "jp" -> "Japanese" + "cn" -> "Chinese" + "es" -> "Spanish" + else -> entry.getStringOrNull("language")?.capitalize() + }, + scanlator = when(entry.getStringOrNull("format")) { + "artistcg" -> "ArtistCG" + "gamecg" -> "GameCG" + "imageset" -> "ImageSet" + else -> entry.getStringOrNull("format")?.capitalize() + }, + source = source + ) + ) + ) + } + + data class SiteTag( + val name: String, + val type: String?, + ) + + override suspend fun getRelatedManga(seed: Manga): List { + val query = """ + queryRecommendations( + type: "chapter" + _id: "${seed.url}" + search: {sortBy:Popular} + page: 1 + size: $size + ) { + chapters { + _id + name + firstPics + } + } + """.trimIndent() + + return apiCall(query) + .getJSONObject("queryRecommendations") + .getJSONArray("chapters") + .toMangaList() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val query = """ + queryChapter( + chapterId: "${chapter.url}" + ) { + pictureUrls { + picCdn + pics + picsS + } + } + """.trimIndent() + + val pages = apiCall(query) + .getJSONObject("queryChapter") + .getJSONArray("pictureUrls") + .getJSONObject(0) + + val cdn = pages.getString("picCdn").let { + if (it.startsWith("http")) { + "$it/" + } else { + "https://${getCdnUrl()}/$it/" + } + } + + val pics = pages.getJSONArray("pics").toJSONList() + val picsS = pages.getJSONArray("picsS").toJSONList() + + return pics.zip(picsS).map { + val img = it.first.getString("url") + val imgS = it.second.getString("url") + MangaPage( + id = generateUid(img), + url = cdn + img, + preview = cdn + imgS, + source = source + ) + } + } + + private suspend fun apiCall(query: String): JSONObject { + return webClient.graphQLQuery("https://api.$domain/api", query).getJSONObject("data") + } + + companion object { + private const val size = 20 + private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + + private fun String.capitalize(): String { + return this.trim().split(" ").joinToString(" ") { word -> + word.replaceFirstChar { + if (it.isLowerCase()) { + it.titlecase( + Locale.getDefault(), + ) + } else { + it.toString() + } + } + } + } + } +} From bf06ab7492be1f0c8f2b114cf4427565c3791303 Mon Sep 17 00:00:00 2001 From: AwkwardPeak7 Date: Tue, 12 Dec 2023 17:27:38 +0500 Subject: [PATCH 2/4] use `SuspendLazy` for fetching cdn host --- .../site/all/NineNineNineHentaiParser.kt | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt index 0a10016f5..46cd808c9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt @@ -19,6 +19,7 @@ import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.SuspendLazy import org.koitharu.kotatsu.parsers.util.domain import org.koitharu.kotatsu.parsers.util.generateUid import org.koitharu.kotatsu.parsers.util.json.getStringOrNull @@ -75,17 +76,13 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang return chain.proceed(newRequest) } - private var cdnUrl: String? = null + private val cdnHost = SuspendLazy(::getUpdatedCdnHost) - private suspend fun getCdnUrl(): String { - if (cdnUrl.isNullOrEmpty()) { - val url = "https://$domain/manga-home" - val response = webClient.httpGet(url).parseHtml() - val cdn = response.selectFirst("img.v-thumbnail")?.attr("data-src") - cdnUrl = cdn?.toHttpUrlOrNull()?.host - } - - return cdnUrl ?: "edge.fast4speed.rsvp" + private suspend fun getUpdatedCdnHost(): String { + val url = "https://$domain/manga-home" + val response = webClient.httpGet(url).parseHtml() + val cdn = response.selectFirst("img.v-thumbnail")?.attr("data-src") + return cdn?.toHttpUrlOrNull()?.host ?: "edge.fast4speed.rsvp" } override suspend fun getAvailableTags(): Set { @@ -212,7 +209,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang coverUrl = when { cover?.startsWith("http") == true -> cover cover == null -> "" - else -> "https://${getCdnUrl()}/$cover" + else -> "https://${cdnHost.get()}/$cover" }, author = null, isNsfw = true, @@ -265,7 +262,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang coverUrl = when { cover?.startsWith("http") == true -> cover cover == null -> "" - else -> "https://${getCdnUrl()}/$cover" + else -> "https://${cdnHost.get()}/$cover" }, author = tags?.filter { it.type == "artist" }?.joinToString { it.name.capitalize() }, isNsfw = true, @@ -356,7 +353,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang if (it.startsWith("http")) { "$it/" } else { - "https://${getCdnUrl()}/$it/" + "https://${cdnHost.get()}/$it/" } } From 0964c02a44a434b6a5277970281f29ba417f90e0 Mon Sep 17 00:00:00 2001 From: AwkwardPeak7 Date: Tue, 12 Dec 2023 18:55:15 +0500 Subject: [PATCH 3/4] largeCover and better chapter branch name --- .../site/all/NineNineNineHentaiParser.kt | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt index 46cd808c9..af3885f4d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt @@ -235,8 +235,12 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang description language pages - firstPics tags + pictureUrls { + picCdn + pics + picsS + } } """.trimIndent() @@ -245,11 +249,20 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang val id = entry.getString("_id") val name = entry.getString("name") - val cover = runCatching { - entry.getJSONArray("firstPics") - .getJSONObject(0) - .getString("url") - }.getOrNull() + val cover = entry.getJSONArray("pictureUrls") + .getJSONObject(0) + .let { pics -> + val cdn = pics.getString("picCdn").let { + if (it.startsWith("http")) { + "$it/" + } else { + "https://${cdnHost.get()}/$it/" + } + } + val img = pics.getJSONArray("pics").getJSONObject(0).getString("url") + val imgS = pics.getJSONArray("picsS").getJSONObject(0).getString("url") + Pair(cdn + imgS, cdn + img) + } val tags = entry.optJSONArray("tags")?.mapJSON { SiteTag( name = it.getString("tagName"), @@ -259,11 +272,8 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang return manga.copy( title = name.replace(shortenTitleRegex, "").trim(), altTitle = name, - coverUrl = when { - cover?.startsWith("http") == true -> cover - cover == null -> "" - else -> "https://${cdnHost.get()}/$cover" - }, + coverUrl = cover.first, + largeCoverUrl = cover.second, author = tags?.filter { it.type == "artist" }?.joinToString { it.name.capitalize() }, isNsfw = true, tags = tags?.mapToSet { @@ -284,12 +294,16 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang uploadDate = kotlin.runCatching { dateFormat.parse(entry.getString("uploadDate"))!!.time }.getOrDefault(0L), - branch = when (entry.getStringOrNull("language")) { - "en" -> "English" - "jp" -> "Japanese" - "cn" -> "Chinese" - "es" -> "Spanish" - else -> entry.getStringOrNull("language")?.capitalize() + branch = entry.getStringOrNull("language")?.let { + val locale = when (it) { + "en" -> Locale.ENGLISH + "jp" -> Locale.JAPANESE + "cn" -> Locale.CHINESE + "es" -> Locale("es") + else -> Locale.ROOT + } + + return@let locale.getDisplayLanguage(locale) }, scanlator = when(entry.getStringOrNull("format")) { "artistcg" -> "ArtistCG" From 3e1b15a23c7bb235008553d97f62f5a84b39f184 Mon Sep 17 00:00:00 2001 From: AwkwardPeak7 Date: Tue, 12 Dec 2023 23:35:07 +0500 Subject: [PATCH 4/4] review changes --- .../site/all/NineNineNineHentaiParser.kt | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt index af3885f4d..0bd23373d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/all/NineNineNineHentaiParser.kt @@ -15,7 +15,6 @@ import org.koitharu.kotatsu.parsers.model.MangaChapter import org.koitharu.kotatsu.parsers.model.MangaListFilter import org.koitharu.kotatsu.parsers.model.MangaPage import org.koitharu.kotatsu.parsers.model.MangaSource -import org.koitharu.kotatsu.parsers.model.MangaState import org.koitharu.kotatsu.parsers.model.MangaTag import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN import org.koitharu.kotatsu.parsers.model.SortOrder @@ -30,6 +29,7 @@ import org.koitharu.kotatsu.parsers.util.mapToSet import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany import org.koitharu.kotatsu.parsers.util.parseHtml import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toCamelCase import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -105,7 +105,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang return tags.mapJSONToSet { val name = it.getString("name") MangaTag( - title = name.capitalize(), + title = name.toCamelCase(), key = name, source = source ) @@ -124,7 +124,9 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang is MangaListFilter.Search -> { getSearchList(page, filter.query, null, null, filter.sortOrder) } - else -> emptyList() + else -> { + getPopularList(page, null) + } } } @@ -217,7 +219,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang publicUrl = "/hchapter/$id".toAbsoluteUrl(domain), tags = emptySet(), source = source, - state = MangaState.FINISHED, + state = null, rating = RATING_UNKNOWN, ) } @@ -274,16 +276,16 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang altTitle = name, coverUrl = cover.first, largeCoverUrl = cover.second, - author = tags?.filter { it.type == "artist" }?.joinToString { it.name.capitalize() }, + author = tags?.filter { it.type == "artist" }?.joinToString { it.name.toCamelCase() }, isNsfw = true, tags = tags?.mapToSet { MangaTag( - title = it.name.capitalize(), + title = it.name.toCamelCase(), key = it.name, source = source ) }.orEmpty(), - state = MangaState.FINISHED, + state = null, description = entry.getStringOrNull("description"), chapters = listOf( MangaChapter( @@ -291,7 +293,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang name = name, number = 1, url = id, - uploadDate = kotlin.runCatching { + uploadDate = runCatching { dateFormat.parse(entry.getString("uploadDate"))!!.time }.getOrDefault(0L), branch = entry.getStringOrNull("language")?.let { @@ -309,7 +311,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang "artistcg" -> "ArtistCG" "gamecg" -> "GameCG" "imageset" -> "ImageSet" - else -> entry.getStringOrNull("format")?.capitalize() + else -> entry.getStringOrNull("format")?.toCamelCase() }, source = source ) @@ -317,7 +319,7 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang ) } - data class SiteTag( + private data class SiteTag( val name: String, val type: String?, ) @@ -394,19 +396,5 @@ internal class NineNineNineHentaiParser(context: MangaLoaderContext) : PagedMang private const val size = 20 private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) - - private fun String.capitalize(): String { - return this.trim().split(" ").joinToString(" ") { word -> - word.replaceFirstChar { - if (it.isLowerCase()) { - it.titlecase( - Locale.getDefault(), - ) - } else { - it.toString() - } - } - } - } } }