Release 0.1.8 (#81)

* Fixes #65 (#73) Remove ktor-http Dependency from ksoup-core by Implementing String.resolveOrNull in Pure Kotlin * Implement KMP CodePoint and remove external dependency (#75) Fixes #74 Create a pure Kotlin implementation for CodePoint & remove the external dependency for CodePoint * fix charbuf NPE * add isUtf8Only flag for charset * update extension function for SourceReader * fix clock millis * add ksoup-lite variant (#77)
fleeksoft · Sep 18, 2024 · b4307aa · b4307aa
1 parent 08dd6c6
commit b4307aa
Show file tree

Hide file tree

Showing 61 changed files with 683 additions and 707 deletions.
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -9,6 +9,7 @@ jobs:
       matrix:
         buildType:
           - "common"
+          - "lite"
           - "kotlinx"
           - "korlibs"
           - "ktor2"

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -18,7 +18,7 @@ jobs:
           { target: windows, os: windows-latest, tasks: mingwX64Test, continueOnError: false },
           { target: linux, os: ubuntu-latest, tasks: linuxX64Test, continueOnError: false },
         ]
-        libBuildType: [ "korlibs", "kotlinx", "okio", "ktor2" ]
+        libBuildType: [ "lite", "korlibs", "kotlinx", "okio", "ktor2" ]
     runs-on: ${{ matrix.config.os }}
     name: Build ${{ matrix.config.target }} with libBuildType=${{ matrix.libBuildType }}
     steps:

diff --git a/README.md b/README.md
@@ -32,7 +32,11 @@ Ksoup is adept at handling all varieties of HTML found in the wild.
 Include the dependency in `commonMain`. Latest version [![Maven Central](https://img.shields.io/maven-central/v/com.fleeksoft.ksoup/ksoup.svg)](https://central.sonatype.com/artifact/com.fleeksoft.ksoup/ksoup)
 
 Ksoup published in four variants. Pick the one that suits your needs and start building!
-1. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 3.0.0-rc-1](https://github.com/ktorio/ktor)**
+1. **This variant is built without any external IO or Network dependencies. Use this if you want to parse HTML from a string.**
+   ```kotlin
+   implementation("com.fleeksoft.ksoup:ksoup-lite:<version>")
+    ```
+2. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 3.0.0-rc-1](https://github.com/ktorio/ktor)**
    ```kotlin
    implementation("com.fleeksoft.ksoup:ksoup:<version>")
 
@@ -41,7 +45,7 @@ Ksoup published in four variants. Pick the one that suits your needs and start b
    implementation("com.fleeksoft.ksoup:ksoup-network:<version>")
     ```
 
-2. **This variant is built with [korlibs-io](https://github.com/korlibs/korlibs-io)**
+3. **This variant is built with [korlibs-io](https://github.com/korlibs/korlibs-io)**
    ```kotlin
    implementation("com.fleeksoft.ksoup:ksoup-korlibs:<version>")
 
@@ -50,15 +54,15 @@ Ksoup published in four variants. Pick the one that suits your needs and start b
    implementation("com.fleeksoft.ksoup:ksoup-network-korlibs:<version>")
     ```
 
-3. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 2.3.12](https://github.com/ktorio/ktor)**
+4. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 2.3.12](https://github.com/ktorio/ktor)**
    ```kotlin
    implementation("com.fleeksoft.ksoup:ksoup-ktor2:<version>")
 
     // Optional: Include only if you need to use network request functions such as
     // Ksoup.parseGetRequest, Ksoup.parseSubmitRequest, and Ksoup.parsePostRequest
    implementation("com.fleeksoft.ksoup:ksoup-network-ktor2:<version>")
     ```
-4. **This variant built with [okio](https://github.com/square/okio) and [Ktor 2.3.12](https://github.com/ktorio/ktor)**
+5. **This variant built with [okio](https://github.com/square/okio) and [Ktor 2.3.12](https://github.com/ktorio/ktor)**
    ```kotlin
    implementation("com.fleeksoft.ksoup:ksoup-okio:<version>")
 

diff --git a/gradle.properties b/gradle.properties
@@ -8,9 +8,9 @@ android.nonTransitiveRClass=true
 kotlin.native.ignoreIncorrectDependencies=true
 kotlin.mpp.enableCInteropCommonization=true
 kotlin.mpp.applyDefaultHierarchyTemplate=false
-# dev, common, kotlinx, korlibs, okio, ktor2
+# dev, common, lite, kotlinx, korlibs, okio, ktor2
 # dev will include all modules in settings.gradle.kts but use kotlinx dep for engine
-libBuildType=okio
+libBuildType=lite
 
 
 SONATYPE_HOST=CENTRAL_PORTAL

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -3,14 +3,13 @@ agp = "8.5.2"
 kotlin = "2.0.20"
 compileSdk = "34"
 minSdk = "21"
-libraryVersion = "0.1.7"
+libraryVersion = "0.1.8"
 ktor = "3.0.0-rc-1"
 ktor2 = "2.3.12"
 coroutines = "1.8.1"
 kotlinxDatetime = "0.6.1"
 kotlinx-io = "0.5.3"
 okio = "3.9.0"
-codepoints = "0.9.0"
 dokka = "1.9.20"
 
 #korlibs = "999.0.0.999" # 999.0.0.999 is local version
@@ -43,7 +42,6 @@ ktor-client-js = { module = "io.ktor:ktor-client-js", version.ref = "ktor" }
 ktor-client-win = { module = "io.ktor:ktor-client-winhttp", version.ref = "ktor" }
 kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinxDatetime" }
 kotlinx-io = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" }
-codepoints = { module = "de.cketti.unicode:kotlin-codepoints-deluxe", version.ref = "codepoints" }
 korlibs-io = { module = "com.soywiz:korlibs-io", version.ref = "korlibs" }
 korlibs-io-network-core = { module = "com.soywiz:korlibs-io-network-core", version.ref = "korlibs" }
 stately-concurrency = { module = "co.touchlab:stately-concurrency", version.ref = "stately" }

diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/engine/KsoupEngine.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/engine/KsoupEngine.kt
@@ -2,14 +2,8 @@ package com.fleeksoft.ksoup.engine
 
 import com.fleeksoft.ksoup.io.Charset
 import com.fleeksoft.ksoup.io.FileSource
-import com.fleeksoft.ksoup.io.SourceReader
 
 interface KsoupEngine {
-    fun urlResolveOrNull(base: String, relUrl: String): String?
-
-    fun openSourceReader(content: String, charset: Charset? = null): SourceReader
-
-    fun openSourceReader(byteArray: ByteArray): SourceReader
 
     fun getUtf8Charset(): Charset
 

diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/Charset.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/Charset.kt
@@ -22,4 +22,6 @@ interface Charset {
 
     fun decode(stringBuilder: StringBuilder, byteArray: ByteArray, start: Int, end: Int): Int
     fun toByteArray(value: String): ByteArray
+
+    fun onlyUtf8(): Boolean = false
 }
diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReaderByteArray.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReaderByteArray.kt
@@ -0,0 +1,68 @@
+package com.fleeksoft.ksoup.io
+
+internal class SourceReaderByteArray(bytes: ByteArray) : SourceReader {
+    private var source: ByteArray = bytes
+    private var currentPosition: Int = 0
+    private var markedPosition: Int? = null
+    private var isClosed: Boolean = false
+
+    override fun mark(readLimit: Long) {
+        markedPosition = currentPosition
+    }
+
+    override fun reset() {
+        isClosed = false
+        markedPosition?.let {
+            currentPosition = it
+            markedPosition = null
+        }
+    }
+
+
+    override fun readBytes(count: Int): ByteArray {
+        val byteArray = ByteArray(count)
+        var i = 0
+        while (exhausted().not() && i < count) {
+            byteArray[i] = source[currentPosition++]
+            i++
+        }
+        return if (i == 0) {
+            byteArrayOf()
+        } else if (i != count) {
+            byteArray.sliceArray(0 until i)
+        } else {
+            byteArray
+        }
+    }
+
+    override fun read(bytes: ByteArray, offset: Int, length: Int): Int {
+        var i = offset
+        while (exhausted().not() && i < length) {
+            bytes[i] = source[currentPosition++]
+            i++
+        }
+        return i
+    }
+
+    override fun readAllBytes(): ByteArray {
+        return readBytes(source.size - currentPosition)
+    }
+
+    override fun exhausted(): Boolean {
+        return currentPosition >= source.size
+    }
+
+    override fun close() {
+//        on reset we need bytes again
+//        source = ByteArray(0)
+//        currentPosition = 0
+//        markedPosition = null
+        isClosed = true
+    }
+
+    override fun readAtMostTo(sink: KByteBuffer, byteCount: Int): Int {
+        val bytes = readBytes(byteCount)
+        sink.writeBytes(bytes, bytes.size)
+        return bytes.size
+    }
+}
diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReaderExt.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReaderExt.kt
@@ -0,0 +1,3 @@
+package com.fleeksoft.ksoup.io
+
+fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderByteArray(byteArray)
diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/EngineExt.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/EngineExt.kt
@@ -1,7 +1,7 @@
 package com.fleeksoft.ksoup
 
-import com.fleeksoft.ksoup.engine.KsoupEngineImpl
 import com.fleeksoft.ksoup.io.SourceReader
+import com.fleeksoft.ksoup.io.from
 import korlibs.io.compression.deflate.GZIP
 import korlibs.io.compression.uncompress
 import korlibs.io.file.VfsFile
@@ -16,8 +16,8 @@ suspend fun VfsFile.openStream(): SourceReader {
         val zipped =
             (byteArray.size == 2 && byteArray[0].toInt() == 31 && byteArray[1].toInt() == -117) // gzip magic bytes 31(0x1f), -117(0x1f)
         if (zipped) {
-            return KsoupEngineImpl.openSourceReader(this.readAsSyncStream().readAll().uncompress(GZIP))
+            return SourceReader.from(this.readAsSyncStream().readAll().uncompress(GZIP))
         }
     }
-    return KsoupEngineImpl.openSourceReader(this.readAll())
+    return SourceReader.from(this.readAll())
 }
diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt
@@ -2,20 +2,8 @@ package com.fleeksoft.ksoup.engine
 
 import com.fleeksoft.ksoup.io.*
 import korlibs.io.lang.Charsets
-import korlibs.io.net.URL
 
 object KsoupEngineImpl : KsoupEngine {
-    override fun urlResolveOrNull(base: String, relUrl: String): String? {
-        return URL.resolveOrNull(base = base, access = relUrl)
-    }
-
-    override fun openSourceReader(content: String, charset: Charset?): SourceReader {
-        return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray())
-    }
-
-    override fun openSourceReader(byteArray: ByteArray): SourceReader {
-        return SourceReader.from(byteArray)
-    }
 
     override fun getUtf8Charset(): Charset {
         return CharsetImpl(Charsets.UTF8)

diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt
@@ -4,7 +4,6 @@ import korlibs.io.file.VfsFile
 import korlibs.io.stream.*
 
 
-fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderImpl(byteArray)
 fun SourceReader.Companion.from(syncStream: SyncStream): SourceReader = SourceReaderImpl(syncStream)
 suspend fun SourceReader.Companion.from(asyncInputStream: AsyncInputStream): SourceReader =
     SourceReaderImpl(asyncInputStream.toAsyncStream().toSyncOrNull() ?: asyncInputStream.readAll().openSync())

diff --git a/ksoup-engine-kotlinx/module.yaml b/ksoup-engine-kotlinx/module.yaml
@@ -10,5 +10,4 @@ aliases:
 dependencies:
   - ../ksoup-engine-common
   - $libs.kotlinx.io: exported
-  - $libs.ktor.io
-  - $libs.ktor.http
+  - $libs.ktor.io
diff --git a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt
@@ -6,18 +6,6 @@ import io.ktor.utils.io.charsets.*
 
 object KsoupEngineImpl : KsoupEngine {
 
-    override fun urlResolveOrNull(base: String, relUrl: String): String? {
-        return URLUtil.urlResolveOrNull(base = base, relUrl = relUrl)
-    }
-
-    override fun openSourceReader(content: String, charset: Charset?): SourceReader {
-        return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray())
-    }
-
-    override fun openSourceReader(byteArray: ByteArray): SourceReader {
-        return SourceReader.from(byteArray)
-    }
-
     override fun getUtf8Charset(): Charset {
         return CharsetImpl(Charsets.UTF_8)
     }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		package com.fleeksoft.ksoup.io

		fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderByteArray(byteArray)