Merge pull request #5 from fleeksoft/develop

Release 0.0.5, fixing issues #2 and #3
fleeksoft · Nov 23, 2023 · 1087e6b · 1087e6b
2 parents 97cee3c + c8dd2b1
commit 1087e6b
Show file tree

Hide file tree

Showing 36 changed files with 247 additions and 279 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -1,33 +1,32 @@
 name: Build
 
-on:
-  push:
+on: 
+  push: 
     branches: [ "main" ]
-  pull_request:
+  pull_request: 
     branches: [ "main" ]
 
-permissions:
+permissions: 
   contents: read
   pages: write
 
-jobs:
-  build:
-    strategy:
-      matrix:
+jobs: 
+  build: 
+    strategy: 
+      matrix: 
         config: [
           { target: android, os: ubuntu-latest, tasks: testDebugUnitTest, continueOnError: false },
-#          { target: apple, os: macos-latest, tasks: iosX64Test iosSimulatorArm64Test macosX64Test macosArm64Test, continueOnError: true },
-          # { target: windows, os: windows-latest, tasks: windowsTest, continueOnError: true }, # https://github.com/square/okio/issues/951
-          # { target: linux, os: ubuntu-latest, tasks: linuxTest, continueOnError: false },
+#          { target: apple, os: macos-latest, tasks: iosX64Test iosSimulatorArm64Test, continueOnError: true },
+#           { target: linux, os: ubuntu-latest, tasks: linuxTest, continueOnError: false },
           { target: desktop, os: ubuntu-latest, tasks: jvmTest, continueOnError: false },
         ]
     runs-on: ${{ matrix.config.os }}
     name: Build ${{ matrix.config.target }}
-    steps:
+    steps: 
       - uses: actions/checkout@v3
       - name: Setup JDK 17
         uses: actions/setup-java@v3
-        with:
+        with: 
           java-version: '17'
           distribution: 'temurin'
 
@@ -38,21 +37,21 @@ jobs:
         continue-on-error: ${{ matrix.config.continueOnError }}
         run: ./gradlew ${{ matrix.config.tasks }}
 
-  release:
+  release: 
     if: ${{ github.event_name != 'pull_request' }}
     runs-on: macos-latest
-    needs:
+    needs: 
       - build
-    environment:
+    environment: 
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
 
-    steps:
+    steps: 
       - uses: actions/checkout@v3
 
       - name: Setup JDK 17
         uses: actions/setup-java@v3
-        with:
+        with: 
           java-version: '17'
           distribution: 'temurin'
 
@@ -83,7 +82,7 @@ jobs:
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v1
-        with:
+        with: 
           path: ${{ github.workspace }}/build/dokka/htmlMultiModule
 
       - name: Release to GitHub Pages

diff --git a/README.md b/README.md
@@ -26,11 +26,11 @@ Ksoup is adept at handling all varieties of HTML found in the wild.
 ### Ksoup is published on Maven Central
 ```Kotlin
 commonMain.dependencies {
-    implementation("com.fleeksoft.ksoup:ksoup:0.0.4")
+    implementation("com.fleeksoft.ksoup:ksoup:0.0.5")
 
 // Optional: Include only if you need to use network request functions such as
 // Ksoup.parseGetRequest, Ksoup.parseSubmitRequest, and Ksoup.parsePostRequest
-    implementation("com.fleeksoft.ksoup:ksoup-network:0.0.4")
+    implementation("com.fleeksoft.ksoup:ksoup-network:0.0.5")
 }
 ```
 

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -3,7 +3,7 @@ agp = "8.1.3"
 kotlin = "1.9.20"
 compileSdk = "34"
 minSdk = "21"
-libraryVersion = "0.0.4"
+libraryVersion = "0.0.5"
 junitJupiter = "5.9.3"
 compose = "1.5.4"
 compose-compiler = "1.5.4"

diff --git a/ksoup-network/build.gradle.kts b/ksoup-network/build.gradle.kts
@@ -41,6 +41,7 @@ kotlin {
         }
         commonTest.dependencies {
             implementation(libs.kotlin.test)
+            implementation(projects.ksoup)
         }
 
         jvmMain.dependencies {

diff --git a/ksoup-network/src/androidMain/kotlin/com/fleeksoft/ksoup/network/Platform.android.kt b/ksoup-network/src/androidMain/kotlin/com/fleeksoft/ksoup/network/Platform.android.kt
@@ -4,9 +4,5 @@ import io.ktor.client.engine.HttpClientEngine
 import io.ktor.client.engine.okhttp.OkHttp
 
 internal actual fun provideHttpClientEngine(): HttpClientEngine {
-    return OkHttp.create {
-        config {
-            followRedirects(true)
-        }
-    }
+    return OkHttp.create {}
 }
diff --git a/ksoup-network/src/commonMain/kotlin/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network/src/commonMain/kotlin/com/fleeksoft/ksoup/network/KsoupNetwork.kt
@@ -12,7 +12,7 @@ import kotlinx.coroutines.runBlocking
  *
  * Use examples:
  *
- *  * `Document doc = Ksoup.parseGetRequest("http://example.com");`
+ *  * `Document doc = Ksoup.parseGetRequest("http://example.com")`
  *
  * @param url URL to connect to. The protocol must be `http` or `https`.
  * @return sane HTML
@@ -23,18 +23,22 @@ public fun Ksoup.parseGetRequest(
     httpRequestBuilder: HttpRequestBuilder.() -> Unit = {},
     parser: Parser = Parser.htmlParser(),
 ): Document {
-    val result: String = runBlocking {
-        NetworkHelper.instance.get(url, httpRequestBuilder = httpRequestBuilder).bodyAsText()
+    var finalUrl: String = url
+    val response: String = runBlocking {
+        val httpResponse = NetworkHelper.instance.get(finalUrl, httpRequestBuilder = httpRequestBuilder)
+//        url can be changed after redirection
+        finalUrl = httpResponse.request.url.toString()
+        httpResponse.bodyAsText()
     }
-    return parse(html = result, parser = parser, baseUri = url)
+    return parse(html = response, parser = parser, baseUri = finalUrl)
 }
 
 /**
  * Use to fetch and parse a HTML page.
  *
  * Use examples:
  *
- *  * `Document doc = Ksoup.parseSubmitRequest("http://example.com", params = mapOf("param1Key" to "param1Value"))
+ *  * `Document doc = Ksoup.parseSubmitRequest("http://example.com", params = mapOf("param1Key" to "param1Value"))`
  *
  * @param url URL to connect to. The protocol must be `http` or `https`.
  * @return sane HTML
@@ -46,15 +50,19 @@ public fun Ksoup.parseSubmitRequest(
     httpRequestBuilder: HttpRequestBuilder.() -> Unit = {},
     parser: Parser = Parser.htmlParser(),
 ): Document {
+    var finalUrl: String = url
     val result: String =
         runBlocking {
-            NetworkHelper.instance.submitForm(
+            val httpResponse = NetworkHelper.instance.submitForm(
                 url = url,
                 params = params,
                 httpRequestBuilder = httpRequestBuilder
-            ).bodyAsText()
+            )
+//            url can be changed after redirection
+            finalUrl = httpResponse.request.url.toString()
+            httpResponse.bodyAsText()
         }
-    return parse(html = result, parser = parser, baseUri = url)
+    return parse(html = result, parser = parser, baseUri = finalUrl)
 }
 
 
@@ -63,7 +71,7 @@ public fun Ksoup.parseSubmitRequest(
  *
  * Use examples:
  *
- *  * `Document doc = Ksoup.parsePostRequest("http://example.com");`
+ *  * `Document doc = Ksoup.parsePostRequest("http://example.com")`
  *
  * @param url URL to connect to. The protocol must be `http` or `https`.
  * @return sane HTML
@@ -74,12 +82,16 @@ public fun Ksoup.parsePostRequest(
     httpRequestBuilder: HttpRequestBuilder.() -> Unit = {},
     parser: Parser = Parser.htmlParser(),
 ): Document {
+    var finalUrl: String = url
     val result: String =
         runBlocking {
-            NetworkHelper.instance.post(
+            val httpResponse = NetworkHelper.instance.post(
                 url = url,
                 httpRequestBuilder = httpRequestBuilder
-            ).bodyAsText()
+            )
+//            url can be changed after redirection
+            finalUrl = httpResponse.request.url.toString()
+            httpResponse.bodyAsText()
         }
-    return parse(html = result, parser = parser, baseUri = url)
+    return parse(html = result, parser = parser, baseUri = finalUrl)
 }
diff --git a/ksoup-network/src/commonMain/kotlin/com/fleeksoft/ksoup/network/NetworkHelper.kt b/ksoup-network/src/commonMain/kotlin/com/fleeksoft/ksoup/network/NetworkHelper.kt
@@ -1,15 +1,23 @@
 package com.fleeksoft.ksoup.network
 
 import io.ktor.client.HttpClient
+import io.ktor.client.plugins.*
+import io.ktor.client.plugins.observer.*
 import io.ktor.client.request.*
 import io.ktor.client.request.forms.submitForm
 import io.ktor.client.statement.HttpResponse
 import io.ktor.http.parameters
+import io.ktor.utils.io.core.*
 
 internal class NetworkHelper(private val client: HttpClient) {
 
     companion object {
-        val instance: NetworkHelper = NetworkHelper(HttpClient(provideHttpClientEngine()))
+        val instance: NetworkHelper = NetworkHelper(HttpClient(provideHttpClientEngine()) {
+            this.followRedirects = true
+            /*this.ResponseObserver {
+                println("headers => ${it.headers}")
+            }*/
+        })
     }
 
     suspend fun get(

diff --git a/ksoup-network/src/commonTest/kotlin/com/fleeksoft/ksoup/KsoupNetworkTest.kt b/ksoup-network/src/commonTest/kotlin/com/fleeksoft/ksoup/KsoupNetworkTest.kt
@@ -0,0 +1,30 @@
+package com.fleeksoft.ksoup
+
+import com.fleeksoft.ksoup.network.parseGetRequest
+import kotlin.test.Ignore
+import kotlin.test.Test
+import kotlin.test.assertNotEquals
+import kotlin.test.assertTrue
+
+class KsoupNetworkTest {
+
+    private val rootUri = "https://aip.dfs.de/BasicVFR/"
+
+    // issue #3 fix
+    @Ignore
+    @Test
+    fun issue3RedirectLocationTest() {
+//        val doc = Jsoup.connect(rootUri).get()
+        val doc = Ksoup.parseGetRequest(rootUri)
+
+        println("doc: ${doc.title()}")
+        val location = doc.location()!!
+        println("location: '$rootUri' --> '$location'")
+        // The output should be "location: 'https://aip.dfs.de/BasicVFR/' --> 'https://aip.dfs.de/BasicVFR/2023NOV16/chapter/daf49d0cf52a85f8ae58cc4f2aed1580.html'"
+
+        assertNotEquals(rootUri, location, "resolveCurrentLink-1")
+        assertTrue(location.startsWith("https://aip.dfs.de/BasicVFR/20"), "resolveCurrentLink-2")
+        assertTrue(location.contains("/chapter/"), "resolveCurrentLink-3")
+    }
+
+}
diff --git a/ksoup-network/src/jvmMain/kotlin/com/fleeksoft/ksoup/network/Platform.jvm.kt b/ksoup-network/src/jvmMain/kotlin/com/fleeksoft/ksoup/network/Platform.jvm.kt
@@ -5,8 +5,5 @@ import io.ktor.client.engine.okhttp.OkHttp
 
 internal actual fun provideHttpClientEngine(): HttpClientEngine {
     return OkHttp.create {
-        config {
-            followRedirects(true)
-        }
     }
 }
diff --git a/ksoup/src/androidMain/kotlin/com/fleeksoft/ksoup/Platform.android.kt b/ksoup/src/androidMain/kotlin/com/fleeksoft/ksoup/Platform.android.kt
@@ -7,7 +7,7 @@ import okio.Path
 import okio.buffer
 
 internal actual fun readGzipFile(file: Path): BufferedSource {
-    val fileSource = FileSystem.SYSTEM.source(file)
+    val fileSource = readFile(file)
     return GzipSource(source = fileSource).buffer()
 }
 

diff --git a/ksoup/src/commonMain/kotlin/com/fleeksoft/ksoup/helper/DataUtil.kt b/ksoup/src/commonMain/kotlin/com/fleeksoft/ksoup/helper/DataUtil.kt
@@ -1,6 +1,7 @@
 package com.fleeksoft.ksoup.helper
 
 import com.fleeksoft.ksoup.UncheckedIOException
+import com.fleeksoft.ksoup.internal.ConstrainableSource
 import com.fleeksoft.ksoup.internal.Normalizer
 import com.fleeksoft.ksoup.internal.StringUtil
 import com.fleeksoft.ksoup.nodes.Comment
@@ -30,7 +31,7 @@ internal object DataUtil {
     val UTF_8: Charset =
         Charsets.UTF_8 // Don't use StandardCharsets, as those only appear in Android API 19, and we target 10.
     private val defaultCharsetName: String = UTF_8.name // used if not found in header or meta charset
-    private const val firstReadBufferSize = 1024 * 5
+    private const val firstReadBufferSize: Long = (1024 * 5).toLong()
     private const val bufferSize: Long = (1024 * 32).toLong()
     private val mimeBoundaryChars =
         "-_1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray()
@@ -155,7 +156,7 @@ internal object DataUtil {
      * @throws IOException on IO error
      */
     @Throws(IOException::class)
-    fun crossStreams(source: ByteArray, outSource: okio.Buffer) {
+    fun crossStreams(source: ByteArray, outSource: Buffer) {
         outSource.write(source)
     }
 
@@ -179,7 +180,7 @@ internal object DataUtil {
 
         val peekedBuffer = bufferReader.getPeek()
         // -1 because we read one more to see if completed. First read is < buffer size, so can't be invalid.
-        val firstBytes: ByteArray = readToByteBuffer(peekedBuffer)
+        val firstBytes: ByteArray = readToByteBuffer(peekedBuffer, firstReadBufferSize - 1)
         val fullyRead = peekedBuffer.exhausted()
         peekedBuffer.close()
 
@@ -303,8 +304,15 @@ internal object DataUtil {
      * @throws IOException if an exception occurs whilst reading from the input stream.
      */
     @Throws(IOException::class)
-    fun readToByteBuffer(bufferReader: BufferedSource): ByteArray {
-        return bufferReader.readByteArray()
+    fun readToByteBuffer(bufferReader: BufferedSource, maxSize: Long): ByteArray {
+        require(maxSize >= 0) {
+            "maxSize must be 0 (unlimited) or larger"
+        }
+        return if (maxSize == 0L) {
+            bufferReader.readByteArray()
+        } else {
+            ConstrainableSource.wrap(BufferReader(bufferReader), maxSize.toInt()).readToByteBuffer(maxSize.toInt()).readByteArray()
+        }
         /*Validate.isTrue(maxSize >= 0, "maxSize must be 0 (unlimited) or larger")
         val calculatedMaxSize: Int =
             if (bufferReader.size() > 0) bufferReader.size().toInt() else maxSize

diff --git a/ksoup/src/commonMain/kotlin/com/fleeksoft/ksoup/internal/StringUtil.kt b/ksoup/src/commonMain/kotlin/com/fleeksoft/ksoup/internal/StringUtil.kt
@@ -308,14 +308,14 @@ internal object StringUtil {
 
     // TODO: replace this
     fun releaseBuilder(sb: StringBuilder): String {
-        var sb: StringBuilder = sb
-        val string: String = sb.toString()
-        if (sb.length > MaxCachedBuilderSize) {
-            sb = StringBuilder(MaxCachedBuilderSize) // make sure it hasn't grown too big
+        var stringBuilder: StringBuilder = sb
+        val string: String = stringBuilder.toString()
+        if (stringBuilder.length > MaxCachedBuilderSize) {
+            stringBuilder = StringBuilder(MaxCachedBuilderSize) // make sure it hasn't grown too big
         } else {
-            sb.clear() // make sure it's emptied on release
+            stringBuilder.clear() // make sure it's emptied on release
         }
-        stringLocalBuilders.addLast(sb)
+        stringLocalBuilders.addLast(stringBuilder)
         while (stringLocalBuilders.size > MaxIdleBuilders) {
             stringLocalBuilders.removeLast()
         }