Merge pull request #51 from fleeksoft/develop

Release 0.1.6-alpha1
fleeksoft · Aug 24, 2024 · dca8d13 · dca8d13
2 parents bb49133 + 6c1a46d
commit dca8d13
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 15 deletions.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -4,7 +4,7 @@ agp = "8.2.2" #kmp max gradle support 8.2 right now
 kotlin = "2.0.10"
 compileSdk = "34"
 minSdk = "21"
-libraryVersion = "0.1.5"
+libraryVersion = "0.1.6-alpha1"
 ktor = "3.0.0-beta-2"
 coroutines = "1.8.1"
 kotlinxDatetime = "0.6.1"

diff --git a/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt
@@ -1,11 +1,15 @@
 package com.fleeksoft.ksoup.network
 
 import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.io.SourceReaderImpl
 import com.fleeksoft.ksoup.nodes.Document
 import com.fleeksoft.ksoup.parser.Parser
 import korlibs.io.async.CIO
 import korlibs.io.net.http.HttpBodyContent
 import korlibs.io.net.http.HttpClient
+import korlibs.io.stream.openSync
+import korlibs.io.stream.toAsyncStream
+import korlibs.io.stream.toSyncOrNull
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 
@@ -33,8 +37,8 @@ public suspend fun Ksoup.parseGetRequest(
     )
 //        url can be changed after redirection
     val finalUrl = httpResponse.headers["location"] ?: url
-    val response = httpResponse.readAllString()
-    return@withContext parse(html = response, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync())
+    return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }
 
 /**
@@ -63,8 +67,8 @@ public suspend fun Ksoup.parseSubmitRequest(
     )
 //            url can be changed after redirection
     val finalUrl = httpResponse.headers["location"] ?: url
-    val response = httpResponse.readAllString()
-    return@withContext parse(html = response, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync())
+    return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }
 
 /**
@@ -93,6 +97,6 @@ public suspend fun Ksoup.parsePostRequest(
     )
 //            url can be changed after redirection
     val finalUrl = httpResponse.headers["location"] ?: url
-    val response = httpResponse.readAllString()
-    return@withContext parse(html = response, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync())
+    return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }
diff --git a/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt
@@ -1,10 +1,14 @@
+@file:OptIn(InternalAPI::class)
+
 package com.fleeksoft.ksoup.network
 
 import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.io.SourceReaderImpl
 import com.fleeksoft.ksoup.nodes.Document
 import com.fleeksoft.ksoup.parser.Parser
 import io.ktor.client.request.*
 import io.ktor.client.statement.*
+import io.ktor.utils.io.*
 
 /**
  * Use to fetch and parse a HTML page.
@@ -25,8 +29,8 @@ public suspend fun Ksoup.parseGetRequest(
     val httpResponse = NetworkHelperKtor.instance.get(url, httpRequestBuilder = httpRequestBuilder)
 //        url can be changed after redirection
     val finalUrl = httpResponse.request.url.toString()
-    val response = httpResponse.bodyAsText()
-    return parse(html = response, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer)
+    return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }
 
 /**
@@ -54,8 +58,8 @@ public suspend fun Ksoup.parseSubmitRequest(
         )
 //            url can be changed after redirection
     val finalUrl = httpResponse.request.url.toString()
-    val result: String = httpResponse.bodyAsText()
-    return parse(html = result, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer)
+    return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }
 
 /**
@@ -74,13 +78,12 @@ public suspend fun Ksoup.parsePostRequest(
     httpRequestBuilder: HttpRequestBuilder.() -> Unit = {},
     parser: Parser = Parser.htmlParser(),
 ): Document {
-    val httpResponse =
-        NetworkHelperKtor.instance.post(
+    val httpResponse = NetworkHelperKtor.instance.post(
             url = url,
             httpRequestBuilder = httpRequestBuilder,
         )
 //            url can be changed after redirection
     val finalUrl = httpResponse.request.url.toString()
-    val result: String = httpResponse.bodyAsText()
-    return parse(html = result, parser = parser, baseUri = finalUrl)
+    val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer)
+    return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl)
 }