From 33be6256ae002d68c48b986af3b8bf2b6b1c28d8 Mon Sep 17 00:00:00 2001 From: Sabeeh Ul Hussnain Date: Sat, 31 Aug 2024 16:26:27 +0500 Subject: [PATCH] Release 0.1.6 (#57) * libBuildType added for multiple variants * test helper functions & minor tweaks * remove skip function * Use source extension functions * minor tweaks * Source extension functions * remove kotlinx dep * add okio module * update test script to support multiple variants * fix platform checks * refactor TestHelper func * wasm target file updated * Add Okio & Ktor2 (#56) * wasm target updated * add ktor2 * add wasm target flag * chore:build configurations fixed * add ksoup-network-ktor2 * update publish scripts * add okio doc * assert message improved * update target scripts * add test clean flag * update gradle props * gradle enableCInteropCommonization * fix jvm args * runTests updated --- .github/workflows/deploy.yml | 23 +-- README.md | 24 ++- gradle.properties | 12 +- gradle/libs.versions.toml | 6 + .../src/com/fleeksoft/ksoup/io/FileSource.kt | 2 + .../com/fleeksoft/ksoup/io/SourceReader.kt | 3 +- ksoup-engine-korlibs/module.yaml | 4 +- .../fleeksoft/ksoup/engine/KsoupEngineImpl.kt | 6 +- .../com/fleeksoft/ksoup/io/FileSourceImpl.kt | 4 +- .../src/com/fleeksoft/ksoup/io/SourceExt.kt | 14 ++ .../ksoup/io/SourceReaderAsyncImpl.kt | 14 +- .../fleeksoft/ksoup/io/SourceReaderImpl.kt | 10 +- .../com/fleeksoft/ksoup/JvmKotlinxMapper.kt | 20 --- .../com/fleeksoft/ksoup/io/SourceExtJvm.kt | 8 + ksoup-engine-kotlinx/module.yaml | 6 +- .../fleeksoft/ksoup/engine/KsoupEngineImpl.kt | 6 +- .../com/fleeksoft/ksoup/io/FileSourceImpl.kt | 4 +- .../src/com/fleeksoft/ksoup/io/SourceExt.kt | 16 ++ .../fleeksoft/ksoup/io/SourceReaderImpl.kt | 6 +- .../com/fleeksoft/ksoup/JvmKotlinxMapper.kt | 19 --- .../com/fleeksoft/ksoup/io/SourceExtJvm.kt | 10 ++ ksoup-engine-ktor2/build.gradle.kts | 37 +++++ ksoup-engine-ktor2/module.yaml | 14 ++ .../fleeksoft/ksoup/engine/KsoupEngineImpl.kt | 32 ++++ .../src/com/fleeksoft/ksoup/engine/URLUtil.kt | 153 ++++++++++++++++++ .../src/com/fleeksoft/ksoup/io/CharsetImpl.kt | 74 +++++++++ .../com/fleeksoft/ksoup/io/FileSourceImpl.kt | 31 ++++ .../src/com/fleeksoft/ksoup/io/SourceExt.kt | 12 ++ .../fleeksoft/ksoup/io/SourceReaderImpl.kt | 67 ++++++++ .../com/fleeksoft/ksoup/io/SourceExtJvm.kt | 10 ++ ksoup-engine-okio/build.gradle.kts | 37 +++++ ksoup-engine-okio/module.yaml | 18 +++ .../fleeksoft/ksoup/engine/KsoupEngineImpl.kt | 32 ++++ .../src/com/fleeksoft/ksoup/engine/URLUtil.kt | 153 ++++++++++++++++++ .../src/com/fleeksoft/ksoup/io/CharsetImpl.kt | 74 +++++++++ .../com/fleeksoft/ksoup/io/FileSourceImpl.kt | 31 ++++ .../src/com/fleeksoft/ksoup/io/IOPlatform.kt | 6 + .../src/com/fleeksoft/ksoup/io/SourceExt.kt | 12 ++ .../fleeksoft/ksoup/io/SourceReaderImpl.kt | 68 ++++++++ .../ksoup/io/IOPlatform.concurrent.kt | 10 ++ .../com/fleeksoft/ksoup/io/IOPlatform.js.kt | 9 ++ .../com/fleeksoft/ksoup/io/SourceExtJvm.kt | 9 ++ ksoup-network-korlibs/build.gradle.kts | 8 + ksoup-network-korlibs/module.yaml | 2 +- .../fleeksoft/ksoup/network/KsoupNetwork.kt | 12 +- ksoup-network-ktor2/build.gradle.kts | 37 +++++ ksoup-network-ktor2/module.yaml | 32 ++++ .../fleeksoft/ksoup/network/KsoupNetwork.kt | 86 ++++++++++ .../ksoup/network/NetworkHelperKtor.kt | 53 ++++++ .../ksoup/network/ProvideHttpClientEngine.kt | 5 + .../network/ProvideHttpClientEngineApple.kt | 8 + .../ksoup/network/KsoupNetworkBlocking.kt | 87 ++++++++++ .../network/ProvideHttpClientEngineJs.kt | 8 + .../network/ProvideHttpClientEngineJvm.kt | 8 + .../network/ProvideHttpClientEngineLinux.kt | 8 + .../network/ProvideHttpClientEngineMingw.kt | 8 + ksoup-network/build.gradle.kts | 8 + ksoup-network/module.yaml | 9 +- .../fleeksoft/ksoup/network/KsoupNetwork.kt | 18 +-- ksoup-test/build.gradle.kts | 46 +++--- ksoup-test/module.yaml | 2 +- .../test/com/fleeksoft/ksoup/TestHelper.kt | 17 +- .../fleeksoft/ksoup/helper/DataUtilTest.kt | 41 +++-- .../fleeksoft/ksoup/integration/ParseTest.kt | 11 +- .../ksoup/issues/GithubIssuesTests.kt | 1 - .../com/fleeksoft/ksoup/nodes/DocumentTest.kt | 2 +- .../com/fleeksoft/ksoup/nodes/ElementIT.kt | 6 +- .../com/fleeksoft/ksoup/nodes/NodeTest.kt | 15 +- .../ksoup/parser/CharacterReaderTest.kt | 7 +- .../fleeksoft/ksoup/parser/HtmlParserTest.kt | 8 +- .../com/fleeksoft/ksoup/parser/ParserIT.kt | 16 +- .../com/fleeksoft/ksoup/DataUtilTestJvm.kt | 12 +- .../ksoup/io/InputStreamReader.test.kt | 3 +- ksoup/build.gradle.kts | 23 ++- ksoup/module.yaml | 2 +- .../com/fleeksoft/ksoup/KsoupJvmExt.kt | 10 +- publishToLocal.sh | 19 --- publishToMaven.sh | 114 +++++++++++-- runAllTests.sh | 25 --- runTests.sh | 126 +++++++++++++++ settings.gradle.kts | 13 +- 81 files changed, 1740 insertions(+), 282 deletions(-) create mode 100644 ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt delete mode 100644 ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt create mode 100644 ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt create mode 100644 ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceExt.kt delete mode 100644 ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt create mode 100644 ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt create mode 100644 ksoup-engine-ktor2/build.gradle.kts create mode 100644 ksoup-engine-ktor2/module.yaml create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/URLUtil.kt create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/CharsetImpl.kt create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceExt.kt create mode 100644 ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt create mode 100644 ksoup-engine-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt create mode 100644 ksoup-engine-okio/build.gradle.kts create mode 100644 ksoup-engine-okio/module.yaml create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/URLUtil.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/io/CharsetImpl.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/io/IOPlatform.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceExt.kt create mode 100644 ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt create mode 100644 ksoup-engine-okio/src@concurrent/com/fleeksoft/ksoup/io/IOPlatform.concurrent.kt create mode 100644 ksoup-engine-okio/src@js/com/fleeksoft/ksoup/io/IOPlatform.js.kt create mode 100644 ksoup-engine-okio/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt create mode 100644 ksoup-network-ktor2/build.gradle.kts create mode 100644 ksoup-network-ktor2/module.yaml create mode 100644 ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt create mode 100644 ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/NetworkHelperKtor.kt create mode 100644 ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/ProvideHttpClientEngine.kt create mode 100644 ksoup-network-ktor2/src@apple/com/fleeksoft/ksoup/network/ProvideHttpClientEngineApple.kt create mode 100644 ksoup-network-ktor2/src@concurrent/com/fleeksoft/ksoup/network/KsoupNetworkBlocking.kt create mode 100644 ksoup-network-ktor2/src@js/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJs.kt create mode 100644 ksoup-network-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJvm.kt create mode 100644 ksoup-network-ktor2/src@linux/com/fleeksoft/ksoup/network/ProvideHttpClientEngineLinux.kt create mode 100644 ksoup-network-ktor2/src@mingw/com/fleeksoft/ksoup/network/ProvideHttpClientEngineMingw.kt delete mode 100755 publishToLocal.sh delete mode 100755 runAllTests.sh create mode 100755 runTests.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 65d4b955..8e9d80e6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,20 +7,21 @@ on: pull_request: branches: [ "release", "develop" ] paths-ignore: [ "**.md" ] + workflow_dispatch: jobs: build: strategy: matrix: config: [ - { target: android, os: ubuntu-latest, tasks: testDebugUnitTest testReleaseUnitTest, continueOnError: false }, - { target: apple, os: macos-latest, tasks: iosX64Test iosSimulatorArm64Test macosX64Test macosArm64Test tvosX64Test tvosSimulatorArm64Test, continueOnError: false }, - { target: js, os: ubuntu-latest, tasks: jsTest wasmTest, continueOnError: false }, - { target: desktop, os: ubuntu-latest, tasks: jvmTest, continueOnError: false }, - { target: windows, os: windows-latest, tasks: mingwX64Test, continueOnError: false }, - { target: linux, os: windows-latest, tasks: linuxX64Test, continueOnError: false }, - ] - libBuildType: [ "korlibs", "kotlinx" ] + { target: android, os: ubuntu-latest, tasks: testDebugUnitTest testReleaseUnitTest, continueOnError: false }, + { target: apple, os: macos-latest, tasks: iosX64Test iosSimulatorArm64Test macosX64Test macosArm64Test tvosX64Test tvosSimulatorArm64Test, continueOnError: false }, + { target: js, os: ubuntu-latest, tasks: jsTest wasmTest, continueOnError: false }, + { target: desktop, os: ubuntu-latest, tasks: jvmTest, continueOnError: false }, + { target: windows, os: windows-latest, tasks: mingwX64Test, continueOnError: false }, + { target: linux, os: windows-latest, tasks: linuxX64Test, continueOnError: false }, + ] + libBuildType: [ "korlibs", "kotlinx", "okio", "ktor2" ] runs-on: ${{ matrix.config.os }} name: Build ${{ matrix.config.target }} with libBuildType=${{ matrix.libBuildType }} steps: @@ -34,9 +35,11 @@ jobs: - name: Setup gradle uses: gradle/gradle-build-action@v2 - - name: Test ${{ matrix.config.target }} targets with libBuildType=${{ matrix.libBuildType }} + - name: Run tests for ${{ matrix.libBuildType }} on ${{ matrix.config.target }} continue-on-error: ${{ matrix.config.continueOnError }} - run: ./gradlew ${{ matrix.config.tasks }} -PlibBuildType=${{ matrix.libBuildType }} --info + run: | + chmod +x ./runTests.sh + ./runTests.sh ${{ matrix.libBuildType }} ${{ matrix.config.tasks }} working-directory: ${{ github.workspace }} # deploy: diff --git a/README.md b/README.md index c0a64cde..1b8ea7b9 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Ksoup is adept at handling all varieties of HTML found in the wild. ### Ksoup is published on Maven Central Include the dependency in `commonMain`. Latest version [![Maven Central](https://img.shields.io/maven-central/v/com.fleeksoft.ksoup/ksoup.svg)](https://central.sonatype.com/artifact/com.fleeksoft.ksoup/ksoup) -Ksoup published in three variants. Pick the one that suits your needs and start building! +Ksoup published in four variants. Pick the one that suits your needs and start building! 1. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 3.0.0-beta-2](https://github.com/ktorio/ktor)** ```kotlin implementation("com.fleeksoft.ksoup:ksoup:") @@ -41,22 +41,30 @@ Ksoup published in three variants. Pick the one that suits your needs and start implementation("com.fleeksoft.ksoup:ksoup-network:") ``` -2. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 2.3.12](https://github.com/ktorio/ktor)** +2. **This variant is built with [korlibs-io](https://github.com/korlibs/korlibs-io)** + ```kotlin + implementation("com.fleeksoft.ksoup:ksoup-korlibs:") + + // Optional: Include only if you need to use network request functions such as + // Ksoup.parseGetRequest, Ksoup.parseSubmitRequest, and Ksoup.parsePostRequest + implementation("com.fleeksoft.ksoup:ksoup-network-korlibs:") + ``` + +3. **This variant built with [kotlinx-io](https://github.com/Kotlin/kotlinx-io) and [Ktor 2.3.12](https://github.com/ktorio/ktor)** ```kotlin implementation("com.fleeksoft.ksoup:ksoup-ktor2:") - + // Optional: Include only if you need to use network request functions such as // Ksoup.parseGetRequest, Ksoup.parseSubmitRequest, and Ksoup.parsePostRequest implementation("com.fleeksoft.ksoup:ksoup-network-ktor2:") ``` - -3. **This variant is built with [korlibs-io](https://github.com/korlibs/korlibs-io)** +4. **This variant built with [okio](https://github.com/square/okio) and [Ktor 2.3.12](https://github.com/ktorio/ktor)** ```kotlin - implementation("com.fleeksoft.ksoup:ksoup-korlibs:") - + implementation("com.fleeksoft.ksoup:ksoup-okio:") + // Optional: Include only if you need to use network request functions such as // Ksoup.parseGetRequest, Ksoup.parseSubmitRequest, and Ksoup.parsePostRequest - implementation("com.fleeksoft.ksoup:ksoup-network-korlibs:") + implementation("com.fleeksoft.ksoup:ksoup-network-ktor2:") ``` **NOTE:** Variants built with kotlinx do not support gzip files. diff --git a/gradle.properties b/gradle.properties index 22e2d051..1f74c807 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,13 +1,13 @@ #Gradle -org.gradle.jvmargs=-Xmx2048M -Dfile.encoding=UTF-8 -Dkotlin.daemon.jvm.options\="-Xmx2048M" -org.gradle.caching=true +org.gradle.jvmargs=-Xmx8192M -Dfile.encoding=UTF-8 -Dkotlin.daemon.jvm.options\="-Xmx8192M" #Kotlin kotlin.code.style=official #Android android.useAndroidX=true android.nonTransitiveRClass=true kotlin.native.ignoreIncorrectDependencies=true - -# dev, kotlinx, korlibs, okio -# dev will include all modules in project -libBuildType=korlibs \ No newline at end of file +kotlin.mpp.enableCInteropCommonization=true +# dev, kotlinx, korlibs, okio, ktor2 +# dev will include all modules in settings.gradle.kts but use kotlinx dep for engine +libBuildType=kotlinx +isWasmEnabled=false \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4a15369b..e3b2df4f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,9 +6,11 @@ compileSdk = "34" minSdk = "21" libraryVersion = "0.1.6-alpha1" ktor = "3.0.0-beta-2" +ktor2 = "2.3.12" coroutines = "1.8.1" kotlinxDatetime = "0.6.1" kotlinx-io = "0.5.3" +okio = "3.9.0" codepoints = "0.9.0" dokka = "1.9.20" @@ -23,6 +25,8 @@ jsoup = "1.18.1" kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-test = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-test", version.ref = "coroutines" } +ktor2-io = { module = "io.ktor:ktor-io", version.ref = "ktor2" } +ktor2-http = { module = "io.ktor:ktor-http", version.ref = "ktor2" } ktor-io = { module = "io.ktor:ktor-io", version.ref = "ktor" } ktor-http = { module = "io.ktor:ktor-http", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } @@ -39,6 +43,8 @@ korlibs-io = { module = "com.soywiz:korlibs-io", version.ref = "korlibs" } korlibs-io-network-core = { module = "com.soywiz:korlibs-io-network-core", version.ref = "korlibs" } stately-concurrent = { module = "co.touchlab:stately-concurrent-collections", version.ref = "stately-concurrent" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +okio = { module = "com.squareup.okio:okio", version.ref = "okio" } +okio-nodefilesystem = { module = "com.squareup.okio:okio-nodefilesystem", version.ref = "okio" } [plugins] androidLibrary = { id = "com.android.library", version.ref = "agp" } diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/FileSource.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/FileSource.kt index 61a5eea6..27abe365 100644 --- a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/FileSource.kt +++ b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/FileSource.kt @@ -4,4 +4,6 @@ interface FileSource { suspend fun toSourceReader(): SourceReader fun getPath(): String fun getFullName(): String + + companion object } \ No newline at end of file diff --git a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReader.kt b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReader.kt index c5f8295e..04609ae1 100644 --- a/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReader.kt +++ b/ksoup-engine-common/src/com/fleeksoft/ksoup/io/SourceReader.kt @@ -1,7 +1,6 @@ package com.fleeksoft.ksoup.io interface SourceReader { - public fun skip(count: Long) public fun mark(readLimit: Long) @@ -18,4 +17,6 @@ interface SourceReader { public fun close() public fun readAtMostTo(sink: KByteBuffer, byteCount: Int): Int + + companion object } \ No newline at end of file diff --git a/ksoup-engine-korlibs/module.yaml b/ksoup-engine-korlibs/module.yaml index b2e1c294..01d04ae8 100644 --- a/ksoup-engine-korlibs/module.yaml +++ b/ksoup-engine-korlibs/module.yaml @@ -1,11 +1,11 @@ product: type: lib - platforms: [jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64] + platforms: [ jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] apply: [ ../common.module-template.yaml ] aliases: - - jvmAndAndroid: [jvm, android] + - jvmAndAndroid: [ jvm, android ] dependencies: - ../ksoup-engine-common diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt index 18a0a0f1..d39476c1 100644 --- a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt +++ b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt @@ -10,11 +10,11 @@ object KsoupEngineImpl : KsoupEngine { } override fun openSourceReader(content: String, charset: Charset?): SourceReader { - return SourceReaderImpl(charset?.toByteArray(content) ?: content.encodeToByteArray()) + return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray()) } override fun openSourceReader(byteArray: ByteArray): SourceReader { - return SourceReaderImpl(byteArray) + return SourceReader.from(byteArray) } override fun getUtf8Charset(): Charset { @@ -26,6 +26,6 @@ object KsoupEngineImpl : KsoupEngine { } override fun pathToFileSource(path: String): FileSource { - return FileSourceImpl(path) + return FileSource.from(path) } } \ No newline at end of file diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt index 30905241..f94d9c43 100644 --- a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt +++ b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt @@ -5,8 +5,8 @@ import korlibs.io.file.VfsFile import korlibs.io.file.fullName import korlibs.io.file.std.uniVfs -class FileSourceImpl : FileSource { - val file: VfsFile +internal class FileSourceImpl : FileSource { + private val file: VfsFile constructor(file: VfsFile) { this.file = file diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt new file mode 100644 index 00000000..24a7fc5f --- /dev/null +++ b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceExt.kt @@ -0,0 +1,14 @@ +package com.fleeksoft.ksoup.io + +import korlibs.io.file.VfsFile +import korlibs.io.stream.* + + +fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderImpl(byteArray) +fun SourceReader.Companion.from(syncStream: SyncStream): SourceReader = SourceReaderImpl(syncStream) +suspend fun SourceReader.Companion.from(asyncInputStream: AsyncInputStream): SourceReader = + SourceReaderImpl(asyncInputStream.toAsyncStream().toSyncOrNull() ?: asyncInputStream.readAll().openSync()) + + +fun FileSource.Companion.from(file: VfsFile): FileSource = FileSourceImpl(file) +fun FileSource.Companion.from(filePath: String): FileSource = FileSourceImpl(filePath) \ No newline at end of file diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderAsyncImpl.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderAsyncImpl.kt index 5476abf0..f57966cc 100644 --- a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderAsyncImpl.kt +++ b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderAsyncImpl.kt @@ -5,10 +5,9 @@ import korlibs.io.stream.SyncInputStream import korlibs.io.stream.markable import korlibs.io.stream.openSync import korlibs.memory.buildByteArray -import kotlin.math.min -class SourceReaderAsyncImpl : SourceReader { +internal class SourceReaderAsyncImpl : SourceReader { private val stream: MarkableSyncInputStream private val buffer: KByteBuffer = KByteBuffer(8192) private var markBuffer: KByteBuffer? = null @@ -20,17 +19,6 @@ class SourceReaderAsyncImpl : SourceReader { constructor(bytes: ByteArray) : this(bytes.openSync()) - override fun skip(count: Long) { - var skippedBuffer = 0 - if (buffer().available() > 0) { - skippedBuffer = min(buffer().available(), count.toInt()) - buffer().skip(skippedBuffer) - } - if (skippedBuffer < count) { - stream.skip(count.toInt() - skippedBuffer) - } - } - override fun mark(readLimit: Long) { markBuffer = buffer.clone() stream.mark(readLimit.toInt()) diff --git a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt index 6236bc5f..959735f2 100644 --- a/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt +++ b/ksoup-engine-korlibs/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt @@ -6,7 +6,7 @@ import korlibs.io.stream.readAll import korlibs.io.stream.readBytes -class SourceReaderImpl : SourceReader { +internal class SourceReaderImpl : SourceReader { private val syncStream: SyncStream constructor(syncStream: SyncStream) { @@ -15,10 +15,6 @@ class SourceReaderImpl : SourceReader { constructor(bytes: ByteArray) : this(bytes.openSync()) - override fun skip(count: Long) { - syncStream.skip(count.toInt()) - } - override fun mark(readLimit: Long) { syncStream.mark(readLimit.toInt()) } @@ -35,10 +31,6 @@ class SourceReaderImpl : SourceReader { return syncStream.read(bytes, offset, length) } - /*override fun availableRead(): Long { - return syncStream.availableRead - }*/ - override fun readAllBytes(): ByteArray { return syncStream.readAll() } diff --git a/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt b/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt deleted file mode 100644 index 7aabaa3a..00000000 --- a/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt +++ /dev/null @@ -1,20 +0,0 @@ -package com.fleeksoft.ksoup - -import com.fleeksoft.ksoup.io.* -import korlibs.io.file.std.toVfs -import korlibs.io.stream.SyncInputStream -import korlibs.io.stream.toAsync -import korlibs.io.stream.toAsyncStream -import korlibs.io.stream.toSyncInputStream -import java.io.File -import java.io.InputStream - -object JvmKotlinxMapper { - fun jvmFileToFileSource(file: File): FileSource { - return FileSourceImpl(file.toVfs()) - } - - fun jvmInputStreamToSourceReader(inputStream: InputStream): SourceReader { - return SourceReaderImpl(inputStream.readAllBytes()) - } -} \ No newline at end of file diff --git a/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt b/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt new file mode 100644 index 00000000..d070cfe5 --- /dev/null +++ b/ksoup-engine-korlibs/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.io + +import korlibs.io.file.std.toVfs +import java.io.File +import java.io.InputStream + +fun FileSource.Companion.from(file: File): FileSource = FileSource.from(file.toVfs()) +fun SourceReader.Companion.from(inputStream: InputStream): SourceReader = SourceReader.from(inputStream.readAllBytes()) \ No newline at end of file diff --git a/ksoup-engine-kotlinx/module.yaml b/ksoup-engine-kotlinx/module.yaml index 8b8656c1..8ab31086 100644 --- a/ksoup-engine-kotlinx/module.yaml +++ b/ksoup-engine-kotlinx/module.yaml @@ -1,14 +1,14 @@ product: type: lib - platforms: [jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64] + platforms: [ jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] apply: [ ../common.module-template.yaml ] aliases: - - jvmAndAndroid: [jvm, android] + - jvmAndAndroid: [ jvm, android ] dependencies: - ../ksoup-engine-common - - $libs.kotlinx.io + - $libs.kotlinx.io: exported - $libs.ktor.io - $libs.ktor.http \ No newline at end of file diff --git a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt index 9b3c7545..5b495d6c 100644 --- a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt +++ b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt @@ -11,11 +11,11 @@ object KsoupEngineImpl : KsoupEngine { } override fun openSourceReader(content: String, charset: Charset?): SourceReader { - return SourceReaderImpl(charset?.toByteArray(content) ?: content.encodeToByteArray()) + return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray()) } override fun openSourceReader(byteArray: ByteArray): SourceReader { - return SourceReaderImpl(byteArray) + return SourceReader.from(byteArray) } override fun getUtf8Charset(): Charset { @@ -27,6 +27,6 @@ object KsoupEngineImpl : KsoupEngine { } override fun pathToFileSource(path: String): FileSource { - return FileSourceImpl(path) + return FileSource.from(path) } } \ No newline at end of file diff --git a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt index 44ffa338..572a8ae0 100644 --- a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt +++ b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt @@ -5,7 +5,7 @@ import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem class FileSourceImpl : FileSource { - val path: Path + private val path: Path private val sourceBuffered by lazy { buffered() } @@ -17,7 +17,7 @@ class FileSourceImpl : FileSource { this.path = Path(filePath) } - override suspend fun toSourceReader(): SourceReader = SourceReaderImpl(sourceBuffered) + override suspend fun toSourceReader(): SourceReader = SourceReader.from(sourceBuffered) private fun FileSourceImpl.buffered() = SystemFileSystem.source(path).buffered() diff --git a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceExt.kt b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceExt.kt new file mode 100644 index 00000000..b370f16c --- /dev/null +++ b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceExt.kt @@ -0,0 +1,16 @@ +@file:OptIn(InternalAPI::class) + +package com.fleeksoft.ksoup.io + +import io.ktor.utils.io.* +import kotlinx.io.Source +import kotlinx.io.files.Path + + +fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderImpl(byteArray) +fun SourceReader.Companion.from(source: Source): SourceReader = SourceReaderImpl(source) +fun SourceReader.Companion.from(bodyChannel: ByteReadChannel): SourceReader = SourceReaderImpl(bodyChannel.readBuffer) + + +fun FileSource.Companion.from(file: Path): FileSource = FileSourceImpl(file) +fun FileSource.Companion.from(filePath: String): FileSource = FileSourceImpl(filePath) \ No newline at end of file diff --git a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt index c3fe8421..706ec963 100644 --- a/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt +++ b/ksoup-engine-kotlinx/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt @@ -4,7 +4,7 @@ import kotlinx.io.Buffer import kotlinx.io.Source import kotlinx.io.readByteArray -class SourceReaderImpl : SourceReader { +internal class SourceReaderImpl : SourceReader { private val source: Source private var sourceMark: Source? = null @@ -18,10 +18,6 @@ class SourceReaderImpl : SourceReader { fun source(): Source = sourceMark ?: source - override fun skip(count: Long) { - source().skip(count) - } - override fun mark(readLimit: Long) { sourceMark = source().peek() } diff --git a/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt b/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt deleted file mode 100644 index 2baa576a..00000000 --- a/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/JvmKotlinxMapper.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.fleeksoft.ksoup - -import com.fleeksoft.ksoup.io.FileSource -import com.fleeksoft.ksoup.io.FileSourceImpl -import com.fleeksoft.ksoup.io.SourceReader -import com.fleeksoft.ksoup.io.SourceReaderImpl -import kotlinx.io.asSource -import kotlinx.io.buffered -import java.io.File -import java.io.InputStream - -object JvmKotlinxMapper { - fun jvmFileToFileSource(file: File): FileSource { - return FileSourceImpl(file.absolutePath) - } - fun jvmInputStreamToSourceReader(inputStream: InputStream): SourceReader { - return SourceReaderImpl(inputStream.asSource().buffered()) - } -} \ No newline at end of file diff --git a/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt b/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt new file mode 100644 index 00000000..09ca50f5 --- /dev/null +++ b/ksoup-engine-kotlinx/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt @@ -0,0 +1,10 @@ +package com.fleeksoft.ksoup.io + +import kotlinx.io.asSource +import kotlinx.io.buffered +import java.io.File +import java.io.InputStream + + +fun FileSource.Companion.from(file: File): FileSource = FileSource.from(file) +fun SourceReader.Companion.from(inputStream: InputStream): SourceReader = SourceReader.from(inputStream.asSource().buffered()) \ No newline at end of file diff --git a/ksoup-engine-ktor2/build.gradle.kts b/ksoup-engine-ktor2/build.gradle.kts new file mode 100644 index 00000000..56ac231f --- /dev/null +++ b/ksoup-engine-ktor2/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + alias(libs.plugins.mavenPublish) +} + +group = "com.fleeksoft.ksoup" +version = libs.versions.libraryVersion.get() + +val artifactId = "ksoup-engine-ktor2" +mavenPublishing { + coordinates("com.fleeksoft.ksoup", artifactId, libs.versions.libraryVersion.get()) + pom { + name.set(artifactId) + description.set("Ksoup is a Kotlin Multiplatform library for working with HTML and XML, and offers an easy-to-use API for URL fetching, data parsing, extraction, and manipulation using DOM and CSS selectors.") + licenses { + license { + name.set("Apache-2.0") + url.set("https://opensource.org/licenses/Apache-2.0") + } + } + url.set("https://github.com/fleeksoft/ksoup") + issueManagement { + system.set("Github") + url.set("https://github.com/fleeksoft/ksoup/issues") + } + scm { + connection.set("https://github.com/fleeksoft/ksoup.git") + url.set("https://github.com/fleeksoft/ksoup") + } + developers { + developer { + name.set("Sabeeh Ul Hussnain Anjum") + email.set("fleeksoft@gmail.com") + organization.set("Fleek Soft") + } + } + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/module.yaml b/ksoup-engine-ktor2/module.yaml new file mode 100644 index 00000000..0cfbffb3 --- /dev/null +++ b/ksoup-engine-ktor2/module.yaml @@ -0,0 +1,14 @@ +product: + type: lib + platforms: [jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64] + +apply: [ ../common.module-template.yaml ] + +aliases: + - jvmAndAndroid: [jvm, android] + +dependencies: + - ../ksoup-engine-common + - $libs.kotlinx.io: exported + - $libs.ktor2.io + - $libs.ktor2.http \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt new file mode 100644 index 00000000..5b495d6c --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt @@ -0,0 +1,32 @@ +package com.fleeksoft.ksoup.engine + +import com.fleeksoft.ksoup.io.* +import com.fleeksoft.ksoup.io.Charset +import io.ktor.utils.io.charsets.* + +object KsoupEngineImpl : KsoupEngine { + + override fun urlResolveOrNull(base: String, relUrl: String): String? { + return URLUtil.urlResolveOrNull(base = base, relUrl = relUrl) + } + + override fun openSourceReader(content: String, charset: Charset?): SourceReader { + return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray()) + } + + override fun openSourceReader(byteArray: ByteArray): SourceReader { + return SourceReader.from(byteArray) + } + + override fun getUtf8Charset(): Charset { + return CharsetImpl(Charsets.UTF_8) + } + + override fun charsetForName(name: String): Charset { + return CharsetImpl(name) + } + + override fun pathToFileSource(path: String): FileSource { + return FileSource.from(path) + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/URLUtil.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/URLUtil.kt new file mode 100644 index 00000000..7ba490bf --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/engine/URLUtil.kt @@ -0,0 +1,153 @@ +package com.fleeksoft.ksoup.engine + +import io.ktor.http.* + +object URLUtil { + private fun String.isValidResourceUrl() = + this.startsWith("http", ignoreCase = true) || this.startsWith("ftp://", ignoreCase = true) || + this.startsWith("ftps://", ignoreCase = true) || + this.startsWith("file:/", ignoreCase = true) || + this.startsWith("//") + + private fun String.isAbsResource(): Boolean = Regex("\\w+:").containsMatchIn(this) + private val validUriScheme: Regex = "^[a-zA-Z][a-zA-Z0-9+-.]*:".toRegex() + + private fun URLBuilder.appendRelativePath(relativePath: String): URLBuilder { + val segments = this.encodedPathSegments.toMutableList() + + val isLastSlash = segments.isNotEmpty() && segments.last() == "" + + // clear / its already joining with / + segments.removeAll { it.isEmpty() } + + val relativePathParts: MutableList = + if (relativePath.contains("?")) { + handleQueryParams(relativePath, "?") + } else if (relativePath.contains("#")) { + handleQueryParams(relativePath, "#") + } else { + relativePath.split("/").toMutableList() + } + + if (relativePathParts.size > 1 && relativePathParts.last() == "/") { + relativePathParts.removeLast() + } + + if (relativePathParts.isNotEmpty() && segments.isNotEmpty() && !isLastSlash && + relativePathParts.first().startsWith("?") + ) { + segments.add("${segments.removeLast()}${relativePathParts.removeFirst()}") + } + +// in files when file://etc/var/message + /var/message = file://var/message +// etc considered as host + + if (this.protocol == URLProtocol.createOrDefault("file")) { + if (relativePathParts.size > 1 && relativePathParts.firstOrNull() == "") { + segments.clear() + // remove first / space + relativePathParts.removeFirst() + this.host = relativePathParts.removeFirst() + } + } + + var isNewPathAdded = false + relativePathParts.forEachIndexed { index, path -> + when (path) { + "" -> { + if (index == 0) { + segments.clear() + } else { + segments.add("") + } + } + + "." -> { +// if its last part and . then append / example: .com/b/c/d + ./g/. = .com/b/c/d/g/ + if (index == relativePathParts.size - 1 && segments[index] != "") { + segments.add("") + } else if (!isLastSlash && !isNewPathAdded) { +// isNewPathAdded use to avoid /b/c/d + g/./h here . will not remove last path because its already added new + segments.removeLastOrNull() + } + } + + ".." -> { + // Clean up last path if exist + if (index == 0 && !isLastSlash) { + segments.removeLastOrNull() + } + if (segments.isNotEmpty()) { + segments.removeLast() + } + } + + else -> { +// remove last trailing path if not query or fragment g.com/a/b to g.com/a + if (index == 0 && segments.isNotEmpty() && + !isLastSlash && !path.startsWith("?") && !path.startsWith("#") + ) { + segments.removeLast() + } + isNewPathAdded = true + segments.add(path) + } + } + } + this.encodedPathSegments = segments + + return this + } + + + private fun handleQueryParams( + relativePath: String, + separator: String, + ): MutableList { + val querySplit = relativePath.split(separator).toMutableList() + val firstQueryPath = querySplit.removeFirst() + val relativePathParts = firstQueryPath.split("/").toMutableList() + if (querySplit.isNotEmpty()) { + relativePathParts.add( + "${relativePathParts.removeLastOrNull() ?: ""}$separator${querySplit.joinToString(separator)}", + ) + } + return relativePathParts + } + + private fun resolve(base: Url, cleanedRelUrl: String): Url { + + if (cleanedRelUrl.isEmpty()) { + return base + } + + if (cleanedRelUrl.isValidResourceUrl()) { + return URLBuilder(cleanedRelUrl).apply { + if (cleanedRelUrl.startsWith("//")) { + protocol = base.protocol + } + }.build() + } + + return URLBuilder( + protocol = base.protocol, + host = base.host, + port = base.port, + pathSegments = base.pathSegments + ).appendRelativePath(cleanedRelUrl).build() + } + + fun urlResolveOrNull(base: String, relUrl: String): String? { + // mailto, tel, geo, about etc.. + if (relUrl.isAbsResource()) { + return relUrl + } + return if (base.isValidResourceUrl()) { + resolve(Url(base), relUrl).toString() + } else if (relUrl.isValidResourceUrl()) { + Url(relUrl).toString() + } else { + if (validUriScheme.matches(relUrl)) relUrl else null + } + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/CharsetImpl.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/CharsetImpl.kt new file mode 100644 index 00000000..f462f9ec --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/CharsetImpl.kt @@ -0,0 +1,74 @@ +package com.fleeksoft.ksoup.io + +import io.ktor.utils.io.charsets.* +import io.ktor.utils.io.core.* +import kotlin.math.max + + +class CharsetImpl(override val name: String) : Charset { + private var charset: io.ktor.utils.io.charsets.Charset = io.ktor.utils.io.charsets.Charset.forName(name) + + constructor(charset: io.ktor.utils.io.charsets.Charset) : this(charset.name) { + this.charset = charset + } + + override fun decode(stringBuilder: StringBuilder, byteArray: ByteArray, start: Int, end: Int): Int { + if (end <= 0) return 0 +// val strSizeBeforeDecode = stringBuilder.length + var incompleteByteIndex = -1 + + val isUtf8 = charset.name.lowercase() == "utf-8" + if (isUtf8) { +// TODO:// may be we can use this for other charsets + val startIndex = if (end > 4) end - 4 else 0 + var i = startIndex + while (i < end) { + val byteLength = guessByteSequenceLength(byteArray[i]) + if (byteLength > 1 && (i + byteLength) > end) { + incompleteByteIndex = i + break + } else { + i += max(byteLength, 1) + } + } + } + val toDecodeSize = if (incompleteByteIndex > 0) { + incompleteByteIndex + } else { + end + } + + val decodedBytes = if (isUtf8) { + stringBuilder.append(byteArray.slice(start until toDecodeSize).toByteArray().decodeToString()) + toDecodeSize - start + } else { + val input = ByteReadPacket(byteArray, start, start + toDecodeSize) + val size = input.remaining + stringBuilder.append(charset.newDecoder().decode(input)) + (size - input.remaining).toInt() + } + /*var invalidBytes = 0 + if (stringBuilder.lastOrNull()?.code == 65533) { + stringBuilder.setLength(stringBuilder.length - 1) + val validDecodedStrBytesSize = stringBuilder.substring(strSizeBeforeDecode).toByteArray(charset).size + invalidBytes = decodedBytes - validDecodedStrBytesSize + } + return decodedBytes - invalidBytes*/ + + return decodedBytes + } + + private fun guessByteSequenceLength(byte: Byte): Int { + return when ((byte.toInt() and 0xFF) shr 4) { + in 0b0000..0b0111 -> 1 + in 0b1100..0b1101 -> 2 + 0b1110 -> 3 + 0b1111 -> 4 + else -> 0 + } + } + + override fun toByteArray(value: String): ByteArray { + return value.toByteArray(charset) + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt new file mode 100644 index 00000000..572a8ae0 --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt @@ -0,0 +1,31 @@ +package com.fleeksoft.ksoup.io + +import kotlinx.io.buffered +import kotlinx.io.files.Path +import kotlinx.io.files.SystemFileSystem + +class FileSourceImpl : FileSource { + private val path: Path + + private val sourceBuffered by lazy { buffered() } + + constructor(file: Path) { + this.path = file + } + + constructor(filePath: String) { + this.path = Path(filePath) + } + + override suspend fun toSourceReader(): SourceReader = SourceReader.from(sourceBuffered) + + private fun FileSourceImpl.buffered() = SystemFileSystem.source(path).buffered() + + override fun getPath(): String { + return this.path.name + } + + override fun getFullName(): String { + return this.path.name + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceExt.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceExt.kt new file mode 100644 index 00000000..8888bf5a --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceExt.kt @@ -0,0 +1,12 @@ +package com.fleeksoft.ksoup.io + +import kotlinx.io.Source +import kotlinx.io.files.Path + + +fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderImpl(byteArray) +fun SourceReader.Companion.from(source: Source): SourceReader = SourceReaderImpl(source) + + +fun FileSource.Companion.from(file: Path): FileSource = FileSourceImpl(file) +fun FileSource.Companion.from(filePath: String): FileSource = FileSourceImpl(filePath) \ No newline at end of file diff --git a/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt new file mode 100644 index 00000000..706ec963 --- /dev/null +++ b/ksoup-engine-ktor2/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt @@ -0,0 +1,67 @@ +package com.fleeksoft.ksoup.io + +import kotlinx.io.Buffer +import kotlinx.io.Source +import kotlinx.io.readByteArray + +internal class SourceReaderImpl : SourceReader { + private val source: Source + private var sourceMark: Source? = null + + constructor(bytes: ByteArray) { + this.source = Buffer().apply { write(bytes) } + } + + constructor(source: Source) { + this.source = source + } + + fun source(): Source = sourceMark ?: source + + override fun mark(readLimit: Long) { + sourceMark = source().peek() + } + + override fun reset() { + sourceMark?.close() + sourceMark = null + } + + override fun readBytes(count: Int): ByteArray { + val byteArray = ByteArray(count) + var i = 0 + while (source().exhausted().not() && i < count) { + byteArray[i] = source().readByte() + i++ + } + return if (i == 0) { + byteArrayOf() + } else if (i != count) { + byteArray.sliceArray(0 until i) + } else { + byteArray + } + } + + override fun read(bytes: ByteArray, offset: Int, length: Int): Int { + return source().readAtMostTo(bytes, offset, endIndex = offset + length) + } + + override fun readAllBytes(): ByteArray { + return source().readByteArray() + } + + override fun exhausted(): Boolean { + return source().exhausted() + } + + override fun close() { + return source().close() + } + + override fun readAtMostTo(sink: KByteBuffer, byteCount: Int): Int { + val bytes = readBytes(byteCount) + sink.writeBytes(bytes, bytes.size) + return bytes.size + } +} \ No newline at end of file diff --git a/ksoup-engine-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt b/ksoup-engine-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt new file mode 100644 index 00000000..09ca50f5 --- /dev/null +++ b/ksoup-engine-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt @@ -0,0 +1,10 @@ +package com.fleeksoft.ksoup.io + +import kotlinx.io.asSource +import kotlinx.io.buffered +import java.io.File +import java.io.InputStream + + +fun FileSource.Companion.from(file: File): FileSource = FileSource.from(file) +fun SourceReader.Companion.from(inputStream: InputStream): SourceReader = SourceReader.from(inputStream.asSource().buffered()) \ No newline at end of file diff --git a/ksoup-engine-okio/build.gradle.kts b/ksoup-engine-okio/build.gradle.kts new file mode 100644 index 00000000..64b90c5d --- /dev/null +++ b/ksoup-engine-okio/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + alias(libs.plugins.mavenPublish) +} + +group = "com.fleeksoft.ksoup" +version = libs.versions.libraryVersion.get() + +val artifactId = "ksoup-engine-okio" +mavenPublishing { + coordinates("com.fleeksoft.ksoup", artifactId, libs.versions.libraryVersion.get()) + pom { + name.set(artifactId) + description.set("Ksoup is a Kotlin Multiplatform library for working with HTML and XML, and offers an easy-to-use API for URL fetching, data parsing, extraction, and manipulation using DOM and CSS selectors.") + licenses { + license { + name.set("Apache-2.0") + url.set("https://opensource.org/licenses/Apache-2.0") + } + } + url.set("https://github.com/fleeksoft/ksoup") + issueManagement { + system.set("Github") + url.set("https://github.com/fleeksoft/ksoup/issues") + } + scm { + connection.set("https://github.com/fleeksoft/ksoup.git") + url.set("https://github.com/fleeksoft/ksoup") + } + developers { + developer { + name.set("Sabeeh Ul Hussnain Anjum") + email.set("fleeksoft@gmail.com") + organization.set("Fleek Soft") + } + } + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/module.yaml b/ksoup-engine-okio/module.yaml new file mode 100644 index 00000000..1af9de96 --- /dev/null +++ b/ksoup-engine-okio/module.yaml @@ -0,0 +1,18 @@ +product: + type: lib + platforms: [ jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + +apply: [ ../common.module-template.yaml ] + +aliases: + - jvmAndAndroid: [ jvm, android ] + - concurrent: [ jvm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + +dependencies: + - ../ksoup-engine-common + - $libs.okio: exported + - $libs.ktor2.io + - $libs.ktor2.http + +dependencies@js: + - $libs.okio.nodefilesystem \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt new file mode 100644 index 00000000..5b495d6c --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/KsoupEngineImpl.kt @@ -0,0 +1,32 @@ +package com.fleeksoft.ksoup.engine + +import com.fleeksoft.ksoup.io.* +import com.fleeksoft.ksoup.io.Charset +import io.ktor.utils.io.charsets.* + +object KsoupEngineImpl : KsoupEngine { + + override fun urlResolveOrNull(base: String, relUrl: String): String? { + return URLUtil.urlResolveOrNull(base = base, relUrl = relUrl) + } + + override fun openSourceReader(content: String, charset: Charset?): SourceReader { + return SourceReader.from(charset?.toByteArray(content) ?: content.encodeToByteArray()) + } + + override fun openSourceReader(byteArray: ByteArray): SourceReader { + return SourceReader.from(byteArray) + } + + override fun getUtf8Charset(): Charset { + return CharsetImpl(Charsets.UTF_8) + } + + override fun charsetForName(name: String): Charset { + return CharsetImpl(name) + } + + override fun pathToFileSource(path: String): FileSource { + return FileSource.from(path) + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/URLUtil.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/URLUtil.kt new file mode 100644 index 00000000..7ba490bf --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/engine/URLUtil.kt @@ -0,0 +1,153 @@ +package com.fleeksoft.ksoup.engine + +import io.ktor.http.* + +object URLUtil { + private fun String.isValidResourceUrl() = + this.startsWith("http", ignoreCase = true) || this.startsWith("ftp://", ignoreCase = true) || + this.startsWith("ftps://", ignoreCase = true) || + this.startsWith("file:/", ignoreCase = true) || + this.startsWith("//") + + private fun String.isAbsResource(): Boolean = Regex("\\w+:").containsMatchIn(this) + private val validUriScheme: Regex = "^[a-zA-Z][a-zA-Z0-9+-.]*:".toRegex() + + private fun URLBuilder.appendRelativePath(relativePath: String): URLBuilder { + val segments = this.encodedPathSegments.toMutableList() + + val isLastSlash = segments.isNotEmpty() && segments.last() == "" + + // clear / its already joining with / + segments.removeAll { it.isEmpty() } + + val relativePathParts: MutableList = + if (relativePath.contains("?")) { + handleQueryParams(relativePath, "?") + } else if (relativePath.contains("#")) { + handleQueryParams(relativePath, "#") + } else { + relativePath.split("/").toMutableList() + } + + if (relativePathParts.size > 1 && relativePathParts.last() == "/") { + relativePathParts.removeLast() + } + + if (relativePathParts.isNotEmpty() && segments.isNotEmpty() && !isLastSlash && + relativePathParts.first().startsWith("?") + ) { + segments.add("${segments.removeLast()}${relativePathParts.removeFirst()}") + } + +// in files when file://etc/var/message + /var/message = file://var/message +// etc considered as host + + if (this.protocol == URLProtocol.createOrDefault("file")) { + if (relativePathParts.size > 1 && relativePathParts.firstOrNull() == "") { + segments.clear() + // remove first / space + relativePathParts.removeFirst() + this.host = relativePathParts.removeFirst() + } + } + + var isNewPathAdded = false + relativePathParts.forEachIndexed { index, path -> + when (path) { + "" -> { + if (index == 0) { + segments.clear() + } else { + segments.add("") + } + } + + "." -> { +// if its last part and . then append / example: .com/b/c/d + ./g/. = .com/b/c/d/g/ + if (index == relativePathParts.size - 1 && segments[index] != "") { + segments.add("") + } else if (!isLastSlash && !isNewPathAdded) { +// isNewPathAdded use to avoid /b/c/d + g/./h here . will not remove last path because its already added new + segments.removeLastOrNull() + } + } + + ".." -> { + // Clean up last path if exist + if (index == 0 && !isLastSlash) { + segments.removeLastOrNull() + } + if (segments.isNotEmpty()) { + segments.removeLast() + } + } + + else -> { +// remove last trailing path if not query or fragment g.com/a/b to g.com/a + if (index == 0 && segments.isNotEmpty() && + !isLastSlash && !path.startsWith("?") && !path.startsWith("#") + ) { + segments.removeLast() + } + isNewPathAdded = true + segments.add(path) + } + } + } + this.encodedPathSegments = segments + + return this + } + + + private fun handleQueryParams( + relativePath: String, + separator: String, + ): MutableList { + val querySplit = relativePath.split(separator).toMutableList() + val firstQueryPath = querySplit.removeFirst() + val relativePathParts = firstQueryPath.split("/").toMutableList() + if (querySplit.isNotEmpty()) { + relativePathParts.add( + "${relativePathParts.removeLastOrNull() ?: ""}$separator${querySplit.joinToString(separator)}", + ) + } + return relativePathParts + } + + private fun resolve(base: Url, cleanedRelUrl: String): Url { + + if (cleanedRelUrl.isEmpty()) { + return base + } + + if (cleanedRelUrl.isValidResourceUrl()) { + return URLBuilder(cleanedRelUrl).apply { + if (cleanedRelUrl.startsWith("//")) { + protocol = base.protocol + } + }.build() + } + + return URLBuilder( + protocol = base.protocol, + host = base.host, + port = base.port, + pathSegments = base.pathSegments + ).appendRelativePath(cleanedRelUrl).build() + } + + fun urlResolveOrNull(base: String, relUrl: String): String? { + // mailto, tel, geo, about etc.. + if (relUrl.isAbsResource()) { + return relUrl + } + return if (base.isValidResourceUrl()) { + resolve(Url(base), relUrl).toString() + } else if (relUrl.isValidResourceUrl()) { + Url(relUrl).toString() + } else { + if (validUriScheme.matches(relUrl)) relUrl else null + } + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/CharsetImpl.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/CharsetImpl.kt new file mode 100644 index 00000000..f462f9ec --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/CharsetImpl.kt @@ -0,0 +1,74 @@ +package com.fleeksoft.ksoup.io + +import io.ktor.utils.io.charsets.* +import io.ktor.utils.io.core.* +import kotlin.math.max + + +class CharsetImpl(override val name: String) : Charset { + private var charset: io.ktor.utils.io.charsets.Charset = io.ktor.utils.io.charsets.Charset.forName(name) + + constructor(charset: io.ktor.utils.io.charsets.Charset) : this(charset.name) { + this.charset = charset + } + + override fun decode(stringBuilder: StringBuilder, byteArray: ByteArray, start: Int, end: Int): Int { + if (end <= 0) return 0 +// val strSizeBeforeDecode = stringBuilder.length + var incompleteByteIndex = -1 + + val isUtf8 = charset.name.lowercase() == "utf-8" + if (isUtf8) { +// TODO:// may be we can use this for other charsets + val startIndex = if (end > 4) end - 4 else 0 + var i = startIndex + while (i < end) { + val byteLength = guessByteSequenceLength(byteArray[i]) + if (byteLength > 1 && (i + byteLength) > end) { + incompleteByteIndex = i + break + } else { + i += max(byteLength, 1) + } + } + } + val toDecodeSize = if (incompleteByteIndex > 0) { + incompleteByteIndex + } else { + end + } + + val decodedBytes = if (isUtf8) { + stringBuilder.append(byteArray.slice(start until toDecodeSize).toByteArray().decodeToString()) + toDecodeSize - start + } else { + val input = ByteReadPacket(byteArray, start, start + toDecodeSize) + val size = input.remaining + stringBuilder.append(charset.newDecoder().decode(input)) + (size - input.remaining).toInt() + } + /*var invalidBytes = 0 + if (stringBuilder.lastOrNull()?.code == 65533) { + stringBuilder.setLength(stringBuilder.length - 1) + val validDecodedStrBytesSize = stringBuilder.substring(strSizeBeforeDecode).toByteArray(charset).size + invalidBytes = decodedBytes - validDecodedStrBytesSize + } + return decodedBytes - invalidBytes*/ + + return decodedBytes + } + + private fun guessByteSequenceLength(byte: Byte): Int { + return when ((byte.toInt() and 0xFF) shr 4) { + in 0b0000..0b0111 -> 1 + in 0b1100..0b1101 -> 2 + 0b1110 -> 3 + 0b1111 -> 4 + else -> 0 + } + } + + override fun toByteArray(value: String): ByteArray { + return value.toByteArray(charset) + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt new file mode 100644 index 00000000..b510bf2f --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/FileSourceImpl.kt @@ -0,0 +1,31 @@ +package com.fleeksoft.ksoup.io + +import okio.Path +import okio.Path.Companion.toPath +import okio.Source + +class FileSourceImpl : FileSource { + val path: Path + + private val sourceBuffered by lazy { buffered() } + + constructor(file: Path) { + this.path = file + } + + constructor(filePath: String) { + this.path = filePath.toPath() + } + + override suspend fun toSourceReader(): SourceReader = SourceReader.from(sourceBuffered) + + private fun FileSourceImpl.buffered(): Source = readFile(path) + + override fun getPath(): String { + return this.path.name + } + + override fun getFullName(): String { + return this.path.name + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/IOPlatform.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/IOPlatform.kt new file mode 100644 index 00000000..fd39d0a0 --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/IOPlatform.kt @@ -0,0 +1,6 @@ +package com.fleeksoft.ksoup.io + +import okio.Path +import okio.Source + +expect fun readFile(file: Path): Source \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceExt.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceExt.kt new file mode 100644 index 00000000..afb8a59a --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceExt.kt @@ -0,0 +1,12 @@ +package com.fleeksoft.ksoup.io + +import okio.Path +import okio.Source + + +fun SourceReader.Companion.from(byteArray: ByteArray): SourceReader = SourceReaderImpl(byteArray) +fun SourceReader.Companion.from(source: Source): SourceReader = SourceReaderImpl(source) + + +fun FileSource.Companion.from(file: Path): FileSource = FileSourceImpl(file) +fun FileSource.Companion.from(filePath: String): FileSource = FileSourceImpl(filePath) \ No newline at end of file diff --git a/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt new file mode 100644 index 00000000..c4f52cae --- /dev/null +++ b/ksoup-engine-okio/src/com/fleeksoft/ksoup/io/SourceReaderImpl.kt @@ -0,0 +1,68 @@ +package com.fleeksoft.ksoup.io + +import okio.Buffer +import okio.BufferedSource +import okio.Source +import okio.buffer + +internal class SourceReaderImpl : SourceReader { + private val source: BufferedSource + private var sourceMark: BufferedSource? = null + + constructor(bytes: ByteArray) { + this.source = Buffer().apply { write(bytes) } + } + + constructor(source: Source) { + this.source = source.buffer() + } + + private fun source(): BufferedSource = sourceMark ?: source + + override fun mark(readLimit: Long) { + sourceMark = source().peek() + } + + override fun reset() { + sourceMark?.close() + sourceMark = null + } + + override fun readBytes(count: Int): ByteArray { + val byteArray = ByteArray(count) + var i = 0 + while (source().exhausted().not() && i < count) { + byteArray[i] = source().readByte() + i++ + } + return if (i == 0) { + byteArrayOf() + } else if (i != count) { + byteArray.sliceArray(0 until i) + } else { + byteArray + } + } + + override fun read(bytes: ByteArray, offset: Int, length: Int): Int { + return source().read(bytes, offset, byteCount = length) + } + + override fun readAllBytes(): ByteArray { + return source().readByteArray() + } + + override fun exhausted(): Boolean { + return source().exhausted() + } + + override fun close() { + return source().close() + } + + override fun readAtMostTo(sink: KByteBuffer, byteCount: Int): Int { + val bytes = readBytes(byteCount) + sink.writeBytes(bytes, bytes.size) + return bytes.size + } +} \ No newline at end of file diff --git a/ksoup-engine-okio/src@concurrent/com/fleeksoft/ksoup/io/IOPlatform.concurrent.kt b/ksoup-engine-okio/src@concurrent/com/fleeksoft/ksoup/io/IOPlatform.concurrent.kt new file mode 100644 index 00000000..696efb49 --- /dev/null +++ b/ksoup-engine-okio/src@concurrent/com/fleeksoft/ksoup/io/IOPlatform.concurrent.kt @@ -0,0 +1,10 @@ +package com.fleeksoft.ksoup.io + +import okio.FileSystem +import okio.Path +import okio.SYSTEM +import okio.Source + +actual fun readFile(file: Path): Source { + return FileSystem.SYSTEM.source(file) +} \ No newline at end of file diff --git a/ksoup-engine-okio/src@js/com/fleeksoft/ksoup/io/IOPlatform.js.kt b/ksoup-engine-okio/src@js/com/fleeksoft/ksoup/io/IOPlatform.js.kt new file mode 100644 index 00000000..47c8e781 --- /dev/null +++ b/ksoup-engine-okio/src@js/com/fleeksoft/ksoup/io/IOPlatform.js.kt @@ -0,0 +1,9 @@ +package com.fleeksoft.ksoup.io + +import okio.NodeJsFileSystem +import okio.Path +import okio.Source + +actual fun readFile(file: Path): Source { + return NodeJsFileSystem.source(file) +} \ No newline at end of file diff --git a/ksoup-engine-okio/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt b/ksoup-engine-okio/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt new file mode 100644 index 00000000..e298c2f7 --- /dev/null +++ b/ksoup-engine-okio/src@jvmAndAndroid/com/fleeksoft/ksoup/io/SourceExtJvm.kt @@ -0,0 +1,9 @@ +package com.fleeksoft.ksoup.io + +import okio.Path.Companion.toOkioPath +import okio.source +import java.io.File +import java.io.InputStream + +fun FileSource.Companion.from(file: File): FileSource = FileSource.from(file.toOkioPath()) +fun SourceReader.Companion.from(inputStream: InputStream): SourceReader = SourceReader.from(inputStream.source()) \ No newline at end of file diff --git a/ksoup-network-korlibs/build.gradle.kts b/ksoup-network-korlibs/build.gradle.kts index 13ea5841..ea1e2783 100644 --- a/ksoup-network-korlibs/build.gradle.kts +++ b/ksoup-network-korlibs/build.gradle.kts @@ -3,6 +3,14 @@ plugins { alias(libs.plugins.mavenPublish) } +val isWasmEnabled = project.findProperty("isWasmEnabled")?.toString()?.toBoolean() ?: false +val libBuildType = project.findProperty("libBuildType")?.toString() +kotlin { + if (isWasmEnabled && libBuildType != "dev") { + wasmJs() + } +} + group = "com.fleeksoft.ksoup" version = libs.versions.libraryVersion.get() diff --git a/ksoup-network-korlibs/module.yaml b/ksoup-network-korlibs/module.yaml index 4191b4dc..5b472d7b 100644 --- a/ksoup-network-korlibs/module.yaml +++ b/ksoup-network-korlibs/module.yaml @@ -1,6 +1,6 @@ product: type: lib - platforms: [ jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + platforms: [ jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] apply: [ ../common.module-template.yaml ] diff --git a/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt index e9d82f6c..95b7ca36 100644 --- a/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt +++ b/ksoup-network-korlibs/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt @@ -1,15 +1,13 @@ package com.fleeksoft.ksoup.network import com.fleeksoft.ksoup.Ksoup -import com.fleeksoft.ksoup.io.SourceReaderImpl +import com.fleeksoft.ksoup.io.SourceReader +import com.fleeksoft.ksoup.io.from import com.fleeksoft.ksoup.nodes.Document import com.fleeksoft.ksoup.parser.Parser import korlibs.io.async.CIO import korlibs.io.net.http.HttpBodyContent import korlibs.io.net.http.HttpClient -import korlibs.io.stream.openSync -import korlibs.io.stream.toAsyncStream -import korlibs.io.stream.toSyncOrNull import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -37,7 +35,7 @@ public suspend fun Ksoup.parseGetRequest( ) // url can be changed after redirection val finalUrl = httpResponse.headers["location"] ?: url - val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync()) + val sourceReader = SourceReader.from(httpResponse.content) return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } @@ -67,7 +65,7 @@ public suspend fun Ksoup.parseSubmitRequest( ) // url can be changed after redirection val finalUrl = httpResponse.headers["location"] ?: url - val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync()) + val sourceReader = SourceReader.from(httpResponse.content) return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } @@ -97,6 +95,6 @@ public suspend fun Ksoup.parsePostRequest( ) // url can be changed after redirection val finalUrl = httpResponse.headers["location"] ?: url - val sourceReader = SourceReaderImpl(httpResponse.content.toAsyncStream().toSyncOrNull() ?: httpResponse.readAllString().openSync()) + val sourceReader = SourceReader.from(httpResponse.content) return@withContext parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } diff --git a/ksoup-network-ktor2/build.gradle.kts b/ksoup-network-ktor2/build.gradle.kts new file mode 100644 index 00000000..71acd901 --- /dev/null +++ b/ksoup-network-ktor2/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + alias(libs.plugins.dokka) + alias(libs.plugins.mavenPublish) +} + +group = "com.fleeksoft.ksoup" +version = libs.versions.libraryVersion.get() + +mavenPublishing { + coordinates("com.fleeksoft.ksoup", "ksoup-network-ktor2", libs.versions.libraryVersion.get()) + pom { + name.set("ksoup-network") + description.set("Ksoup is a Kotlin Multiplatform library for working with HTML and XML, and offers an easy-to-use API for URL fetching, data parsing, extraction, and manipulation using DOM and CSS selectors.") + licenses { + license { + name.set("Apache-2.0") + url.set("https://opensource.org/licenses/Apache-2.0") + } + } + url.set("https://github.com/fleeksoft/ksoup") + issueManagement { + system.set("Github") + url.set("https://github.com/fleeksoft/ksoup/issues") + } + scm { + connection.set("https://github.com/fleeksoft/ksoup.git") + url.set("https://github.com/fleeksoft/ksoup") + } + developers { + developer { + name.set("Sabeeh Ul Hussnain Anjum") + email.set("fleeksoft@gmail.com") + organization.set("Fleek Soft") + } + } + } +} \ No newline at end of file diff --git a/ksoup-network-ktor2/module.yaml b/ksoup-network-ktor2/module.yaml new file mode 100644 index 00000000..3be64cbe --- /dev/null +++ b/ksoup-network-ktor2/module.yaml @@ -0,0 +1,32 @@ +product: + type: lib + platforms: [ jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + +apply: [ ../common.module-template.yaml ] + +aliases: + - jvmAndAndroid: [ jvm, android ] + - concurrent: [ jvm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + +repositories: + - mavenLocal + +dependencies: + - $libs.kotlinx.coroutines.core + - ../ksoup: compile-only + - $libs.ktor.client.core: exported + +dependencies@jvmAndAndroid: + - $libs.ktor.client.okhttp + +dependencies@apple: + - $libs.ktor.client.darwin + +dependencies@js: + - $libs.ktor.client.js + +dependencies@mingw: + - $libs.ktor.client.win + +dependencies@linux: + - $libs.ktor.client.cio \ No newline at end of file diff --git a/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt new file mode 100644 index 00000000..a20e2c49 --- /dev/null +++ b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt @@ -0,0 +1,86 @@ +package com.fleeksoft.ksoup.network + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.nodes.Document +import com.fleeksoft.ksoup.parser.Parser +import io.ktor.client.request.* +import io.ktor.client.statement.* + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parseGetRequest("http://example.com")` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public suspend fun Ksoup.parseGetRequest( + url: String, + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + parser: Parser = Parser.htmlParser(), +): Document { + val httpResponse = NetworkHelperKtor.instance.get(url, httpRequestBuilder = httpRequestBuilder) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val response = httpResponse.bodyAsText() + return parse(html = response, parser = parser, baseUri = finalUrl) +} + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parseSubmitRequest("http://example.com", params = mapOf("param1Key" to "param1Value"))` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public suspend fun Ksoup.parseSubmitRequest( + url: String, + params: Map = emptyMap(), + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + parser: Parser = Parser.htmlParser(), +): Document { + val httpResponse = + NetworkHelperKtor.instance.submitForm( + url = url, + params = params, + httpRequestBuilder = httpRequestBuilder, + ) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val result: String = httpResponse.bodyAsText() + return parse(html = result, parser = parser, baseUri = finalUrl) +} + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parsePostRequest("http://example.com")` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public suspend fun Ksoup.parsePostRequest( + url: String, + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + parser: Parser = Parser.htmlParser(), +): Document { + val httpResponse = + NetworkHelperKtor.instance.post( + url = url, + httpRequestBuilder = httpRequestBuilder, + ) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val result: String = httpResponse.bodyAsText() + return parse(html = result, parser = parser, baseUri = finalUrl) +} diff --git a/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/NetworkHelperKtor.kt b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/NetworkHelperKtor.kt new file mode 100644 index 00000000..cb95db6b --- /dev/null +++ b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/NetworkHelperKtor.kt @@ -0,0 +1,53 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.* +import io.ktor.client.request.* +import io.ktor.client.request.forms.* +import io.ktor.client.statement.* +import io.ktor.http.* + +public class NetworkHelperKtor(private val client: HttpClient) { + companion object { + val instance: NetworkHelperKtor = NetworkHelperKtor( + HttpClient(provideHttpClientEngine()) { + this.followRedirects = true + }, + ) + } + + suspend fun get( + url: String, + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + ): HttpResponse { + return client.get(url) { + httpRequestBuilder() + } + } + + suspend fun submitForm( + url: String, + params: Map, + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + ): HttpResponse { + return client.submitForm( + url = url, + formParameters = + parameters { + params.forEach { (key, value) -> + append(key, value) + } + }, + ) { + httpRequestBuilder() + } + } + + suspend fun post( + url: String, + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, + ): HttpResponse { + return client.post(url) { + httpRequestBuilder() + } + } +} diff --git a/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/ProvideHttpClientEngine.kt b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/ProvideHttpClientEngine.kt new file mode 100644 index 00000000..f95f59e8 --- /dev/null +++ b/ksoup-network-ktor2/src/com/fleeksoft/ksoup/network/ProvideHttpClientEngine.kt @@ -0,0 +1,5 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* + +expect fun provideHttpClientEngine(): HttpClientEngine \ No newline at end of file diff --git a/ksoup-network-ktor2/src@apple/com/fleeksoft/ksoup/network/ProvideHttpClientEngineApple.kt b/ksoup-network-ktor2/src@apple/com/fleeksoft/ksoup/network/ProvideHttpClientEngineApple.kt new file mode 100644 index 00000000..9f85fe71 --- /dev/null +++ b/ksoup-network-ktor2/src@apple/com/fleeksoft/ksoup/network/ProvideHttpClientEngineApple.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* +import io.ktor.client.engine.darwin.Darwin + +actual fun provideHttpClientEngine(): HttpClientEngine { + return Darwin.create() +} \ No newline at end of file diff --git a/ksoup-network-ktor2/src@concurrent/com/fleeksoft/ksoup/network/KsoupNetworkBlocking.kt b/ksoup-network-ktor2/src@concurrent/com/fleeksoft/ksoup/network/KsoupNetworkBlocking.kt new file mode 100644 index 00000000..72df825c --- /dev/null +++ b/ksoup-network-ktor2/src@concurrent/com/fleeksoft/ksoup/network/KsoupNetworkBlocking.kt @@ -0,0 +1,87 @@ +package com.fleeksoft.ksoup.network + +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.nodes.Document +import com.fleeksoft.ksoup.parser.Parser +import io.ktor.client.request.* +import io.ktor.client.statement.* +import kotlinx.coroutines.runBlocking + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parseGetRequest("http://example.com")` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public fun Ksoup.parseGetRequestBlocking( + url: String, + parser: Parser = Parser.htmlParser(), + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, +): Document = runBlocking { + val httpResponse = NetworkHelperKtor.instance.get(url, httpRequestBuilder = httpRequestBuilder) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val response = httpResponse.bodyAsText() + return@runBlocking parse(html = response, parser = parser, baseUri = finalUrl) +} + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parseSubmitRequest("http://example.com", params = mapOf("param1Key" to "param1Value"))` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public fun Ksoup.parseSubmitRequestBlocking( + url: String, + params: Map = emptyMap(), + parser: Parser = Parser.htmlParser(), + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, +): Document = runBlocking { + val httpResponse = + NetworkHelperKtor.instance.submitForm( + url = url, + params = params, + httpRequestBuilder = httpRequestBuilder, + ) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val result: String = httpResponse.bodyAsText() + return@runBlocking parse(html = result, parser = parser, baseUri = finalUrl) +} + +/** + * Use to fetch and parse a HTML page. + * + * Use examples: + * + * * `Document doc = Ksoup.parsePostRequest("http://example.com")` + * + * @param url URL to connect to. The protocol must be `http` or `https`. + * @return sane HTML + * + */ +public fun Ksoup.parsePostRequestBlocking( + url: String, + parser: Parser = Parser.htmlParser(), + httpRequestBuilder: HttpRequestBuilder.() -> Unit = {}, +): Document = runBlocking { + val httpResponse = + NetworkHelperKtor.instance.post( + url = url, + httpRequestBuilder = httpRequestBuilder, + ) +// url can be changed after redirection + val finalUrl = httpResponse.request.url.toString() + val result: String = httpResponse.bodyAsText() + return@runBlocking parse(html = result, parser = parser, baseUri = finalUrl) +} diff --git a/ksoup-network-ktor2/src@js/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJs.kt b/ksoup-network-ktor2/src@js/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJs.kt new file mode 100644 index 00000000..9851284d --- /dev/null +++ b/ksoup-network-ktor2/src@js/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJs.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* +import io.ktor.client.engine.js.* + +actual fun provideHttpClientEngine(): HttpClientEngine { + return Js.create() +} \ No newline at end of file diff --git a/ksoup-network-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJvm.kt b/ksoup-network-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJvm.kt new file mode 100644 index 00000000..29281078 --- /dev/null +++ b/ksoup-network-ktor2/src@jvmAndAndroid/com/fleeksoft/ksoup/network/ProvideHttpClientEngineJvm.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* +import io.ktor.client.engine.okhttp.OkHttp + +actual fun provideHttpClientEngine(): HttpClientEngine { + return OkHttp.create() +} \ No newline at end of file diff --git a/ksoup-network-ktor2/src@linux/com/fleeksoft/ksoup/network/ProvideHttpClientEngineLinux.kt b/ksoup-network-ktor2/src@linux/com/fleeksoft/ksoup/network/ProvideHttpClientEngineLinux.kt new file mode 100644 index 00000000..32b570e3 --- /dev/null +++ b/ksoup-network-ktor2/src@linux/com/fleeksoft/ksoup/network/ProvideHttpClientEngineLinux.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* +import io.ktor.client.engine.cio.* + +actual fun provideHttpClientEngine(): HttpClientEngine { + return CIO.create() +} \ No newline at end of file diff --git a/ksoup-network-ktor2/src@mingw/com/fleeksoft/ksoup/network/ProvideHttpClientEngineMingw.kt b/ksoup-network-ktor2/src@mingw/com/fleeksoft/ksoup/network/ProvideHttpClientEngineMingw.kt new file mode 100644 index 00000000..61ae2030 --- /dev/null +++ b/ksoup-network-ktor2/src@mingw/com/fleeksoft/ksoup/network/ProvideHttpClientEngineMingw.kt @@ -0,0 +1,8 @@ +package com.fleeksoft.ksoup.network + +import io.ktor.client.engine.* +import io.ktor.client.engine.winhttp.* + +actual fun provideHttpClientEngine(): HttpClientEngine { + return WinHttp.create() +} \ No newline at end of file diff --git a/ksoup-network/build.gradle.kts b/ksoup-network/build.gradle.kts index 7db02e3f..76b28bb7 100644 --- a/ksoup-network/build.gradle.kts +++ b/ksoup-network/build.gradle.kts @@ -3,6 +3,14 @@ plugins { alias(libs.plugins.mavenPublish) } +val isWasmEnabled = project.findProperty("isWasmEnabled")?.toString()?.toBoolean() ?: false +val libBuildType = project.findProperty("libBuildType")?.toString() +kotlin { + if (isWasmEnabled && libBuildType != "dev") { + wasmJs() + } +} + group = "com.fleeksoft.ksoup" version = libs.versions.libraryVersion.get() diff --git a/ksoup-network/module.yaml b/ksoup-network/module.yaml index 65bff510..7f4d9bd0 100644 --- a/ksoup-network/module.yaml +++ b/ksoup-network/module.yaml @@ -1,13 +1,12 @@ product: type: lib - platforms: [ jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + platforms: [ jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] apply: [ ../common.module-template.yaml ] aliases: - jvmAndAndroid: [ jvm, android ] - concurrent: [ jvm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] - - jsAndWasm: [js, wasm] repositories: - mavenLocal @@ -15,7 +14,6 @@ repositories: dependencies: - $libs.kotlinx.coroutines.core - ../ksoup: compile-only - - ../ksoup-engine-kotlinx - $libs.ktor.client.core: exported dependencies@jvmAndAndroid: @@ -24,7 +22,10 @@ dependencies@jvmAndAndroid: dependencies@apple: - $libs.ktor.client.darwin -dependencies@jsAndWasm: +dependencies@js: + - $libs.ktor.client.js + +dependencies@wasm: - $libs.ktor.client.js dependencies@mingw: diff --git a/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt b/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt index b4ea7259..f9fec9ed 100644 --- a/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt +++ b/ksoup-network/src/com/fleeksoft/ksoup/network/KsoupNetwork.kt @@ -1,14 +1,12 @@ -@file:OptIn(InternalAPI::class) - package com.fleeksoft.ksoup.network import com.fleeksoft.ksoup.Ksoup -import com.fleeksoft.ksoup.io.SourceReaderImpl +import com.fleeksoft.ksoup.io.SourceReader +import com.fleeksoft.ksoup.io.from import com.fleeksoft.ksoup.nodes.Document import com.fleeksoft.ksoup.parser.Parser import io.ktor.client.request.* import io.ktor.client.statement.* -import io.ktor.utils.io.* /** * Use to fetch and parse a HTML page. @@ -29,7 +27,7 @@ public suspend fun Ksoup.parseGetRequest( val httpResponse = NetworkHelperKtor.instance.get(url, httpRequestBuilder = httpRequestBuilder) // url can be changed after redirection val finalUrl = httpResponse.request.url.toString() - val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer) + val sourceReader = SourceReader.from(httpResponse.bodyAsChannel()) return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } @@ -58,7 +56,7 @@ public suspend fun Ksoup.parseSubmitRequest( ) // url can be changed after redirection val finalUrl = httpResponse.request.url.toString() - val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer) + val sourceReader = SourceReader.from(httpResponse.bodyAsChannel()) return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } @@ -79,11 +77,11 @@ public suspend fun Ksoup.parsePostRequest( parser: Parser = Parser.htmlParser(), ): Document { val httpResponse = NetworkHelperKtor.instance.post( - url = url, - httpRequestBuilder = httpRequestBuilder, - ) + url = url, + httpRequestBuilder = httpRequestBuilder, + ) // url can be changed after redirection val finalUrl = httpResponse.request.url.toString() - val sourceReader = SourceReaderImpl(httpResponse.bodyAsChannel().readBuffer) + val sourceReader = SourceReader.from(httpResponse.bodyAsChannel()) return parse(sourceReader = sourceReader, parser = parser, baseUri = finalUrl) } diff --git a/ksoup-test/build.gradle.kts b/ksoup-test/build.gradle.kts index 7d783519..95f14014 100644 --- a/ksoup-test/build.gradle.kts +++ b/ksoup-test/build.gradle.kts @@ -1,9 +1,32 @@ -val libBuildType = project.findProperty("libBuildType")?.toString() plugins { alias(libs.plugins.power.assert) } + val rootPath = "generated/kotlin" val isGithubActions: Boolean = System.getenv("GITHUB_ACTIONS")?.toBoolean() == true + +val libBuildType = project.findProperty("libBuildType")?.toString() +val isWasmEnabled = project.findProperty("isWasmEnabled")?.toString()?.toBoolean() ?: false +kotlin { + js(IR) { + browser { + testTask { + useMocha { + timeout = "9s" + } + } + } + } + if (isWasmEnabled && libBuildType != "dev" && (libBuildType == "korlibs" || libBuildType == "kotlinx")) { + wasmJs() + } + sourceSets { + commonTest { + this.kotlin.srcDir(layout.buildDirectory.file(rootPath)) + } + } +} + val generateBuildConfigFile: Task by tasks.creating { group = "build setup" val file = layout.buildDirectory.file("$rootPath/BuildConfig.kt") @@ -17,7 +40,11 @@ val generateBuildConfigFile: Task by tasks.creating { object BuildConfig { const val PROJECT_ROOT: String = "${rootProject.rootDir.absolutePath.replace("\\", "\\\\")}" const val isGithubActions: Boolean = $isGithubActions + const val libBuildType: String = "$libBuildType" const val isKotlinx: Boolean = ${libBuildType == "kotlinx" || libBuildType == "dev"} + const val isKorlibs: Boolean = ${libBuildType == "korlibs"} + const val isOkio: Boolean = ${libBuildType == "okio"} + const val isKtor2: Boolean = ${libBuildType == "ktor2"} } """.trimIndent() file.get().asFile.writeText(content) @@ -28,21 +55,4 @@ tasks.configureEach { if (name != generateBuildConfigFile.name && !name.contains("publish", ignoreCase = true)) { dependsOn(generateBuildConfigFile.name) } -} - -kotlin { - sourceSets { - commonTest { - this.kotlin.srcDir(layout.buildDirectory.file(rootPath)) - } - } - js(IR) { - browser { - testTask { - useMocha { - timeout = "9s" - } - } - } - } } \ No newline at end of file diff --git a/ksoup-test/module.yaml b/ksoup-test/module.yaml index 1da14f68..8303702e 100644 --- a/ksoup-test/module.yaml +++ b/ksoup-test/module.yaml @@ -1,6 +1,6 @@ product: type: lib - platforms: [ jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] + platforms: [ jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64 ] apply: [ ../common.module-template.yaml ] diff --git a/ksoup-test/test/com/fleeksoft/ksoup/TestHelper.kt b/ksoup-test/test/com/fleeksoft/ksoup/TestHelper.kt index ad50aefb..4f583942 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/TestHelper.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/TestHelper.kt @@ -24,9 +24,9 @@ object TestHelper { } fun getResourceAbsolutePath(resourceName: String, absForWindows: Boolean = true): String { - if (Platform.isWindows() && !BuildConfig.isKotlinx && absForWindows) { + if (Platform.isWindows() && BuildConfig.isKorlibs && absForWindows) { return "../../../../testResources/$resourceName" - } else if (Platform.isJsOrWasm()) { + } else if ((Platform.isJsOrWasm() && BuildConfig.isKorlibs) || (Platform.isWasmJs())) { return "https://raw.githubusercontent.com/fleeksoft/ksoup/release/ksoup-test/testResources/$resourceName" } return "${BuildConfig.PROJECT_ROOT}/ksoup-test/testResources/$resourceName" @@ -51,7 +51,7 @@ object TestHelper { private suspend fun readFile(resource: String): SourceReader { val abs = getResourceAbsolutePath(resource, absForWindows = false) - val bytes = if (Platform.isJsOrWasm()) { + val bytes = if (abs.startsWith("https://", ignoreCase = true)) { abs.uniVfs.readAll() } else { SystemFileSystem.source(Path(abs)).buffered().readByteArray() @@ -61,11 +61,18 @@ object TestHelper { private suspend fun readGzipFile(resource: String): SourceReader { val abs = getResourceAbsolutePath(resource, absForWindows = false) - val bytes = if (Platform.isJsOrWasm()) { + val bytes = if (abs.startsWith("https://", ignoreCase = true)) { abs.uniVfs.readAll() } else { SystemFileSystem.source(Path(abs)).buffered().readByteArray() } return bytes.uncompress(GZIP).openSourceReader() } -} + + fun isGzipSupported(): Boolean = BuildConfig.isKorlibs + fun isUtf16Supported(): Boolean = !((BuildConfig.isKotlinx || BuildConfig.isOkio || BuildConfig.isKtor2) && Platform.isJsOrWasm()) + fun isUtf32Supported(): Boolean = !(Platform.isJsOrWasm() || Platform.isWindows() || Platform.isLinux()) + fun isEUCKRSupported(): Boolean = !(Platform.isJsOrWasm() || Platform.isApple() || Platform.isWindows()) + fun isGB2312Supported(): Boolean = !(Platform.isApple() || Platform.isWindows() || ((BuildConfig.isKotlinx || BuildConfig.isOkio || BuildConfig.isKtor2) && Platform.isJsOrWasm())) + fun canReadResourceFile(): Boolean = !Platform.isWasmJs() || BuildConfig.isKorlibs +} \ No newline at end of file diff --git a/ksoup-test/test/com/fleeksoft/ksoup/helper/DataUtilTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/helper/DataUtilTest.kt index 33533359..0e98c27c 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/helper/DataUtilTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/helper/DataUtilTest.kt @@ -1,6 +1,7 @@ package com.fleeksoft.ksoup.helper -import com.fleeksoft.ksoup.* +import com.fleeksoft.ksoup.Ksoup +import com.fleeksoft.ksoup.TestHelper import com.fleeksoft.ksoup.io.SourceReader import com.fleeksoft.ksoup.nodes.Document import com.fleeksoft.ksoup.parser.Parser @@ -54,7 +55,6 @@ class DataUtilTest { charsetName = "UTF-8", parser = Parser.htmlParser(), ) - println("doc: $doc") assertEquals("One", doc.head().text()) } @@ -128,7 +128,7 @@ class DataUtilTest { @Test fun secondMetaElementWithContentTypeContainsCharsetParameter() { - if (Platform.isJsOrWasm() || Platform.isApple() || Platform.isWindows()) { + if (!TestHelper.isEUCKRSupported()) { // FIXME: euc-kr charset not supported return } @@ -148,11 +148,10 @@ class DataUtilTest { @Test fun firstMetaElementWithCharsetShouldBeUsedForDecoding() { - val html = - "" + - "" + - "" + - "Übergrößenträger" + val html = "" + + "" + + "" + + "Übergrößenträger" val docByteArrayCharset: Document = DataUtil.parseInputSource( sourceReader = dataToStream(data = html, charset = "iso-8859-1"), @@ -166,7 +165,7 @@ class DataUtilTest { @Test fun supportsBOMinFiles() = runTest { - if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + if (!TestHelper.isUtf16Supported()) { // FIXME: UTF-16 charset not supported return@runTest } @@ -180,7 +179,7 @@ class DataUtilTest { assertContains(doc.title(), "UTF-16LE") assertContains(doc.text(), "가각갂갃간갅") - if (Platform.isJsOrWasm() || Platform.isWindows() || Platform.isLinux()) { + if (!TestHelper.isUtf32Supported()) { // FIXME: UTF-32 charset not supported return@runTest } @@ -197,7 +196,7 @@ class DataUtilTest { @Test fun streamerSupportsBOMinFiles() = runTest { - if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + if (!TestHelper.isUtf16Supported()) { // FIXME: UTF-16 charset not supported return@runTest } @@ -216,7 +215,7 @@ class DataUtilTest { assertContains(doc.title(), "UTF-16LE") assertContains(doc.text(), "가각갂갃간갅") - if (Platform.isJsOrWasm() || Platform.isWindows() || Platform.isLinux()) { + if (!TestHelper.isUtf32Supported()) { // FIXME: UTF-32 charset not supported return@runTest } @@ -252,7 +251,7 @@ class DataUtilTest { @Test fun supportsZippedUTF8BOM() = runTest { val resourceName = "bomtests/bom_utf8.html.gz" - val doc: Document = if (BuildConfig.isKotlinx) { + val doc: Document = if (!TestHelper.isGzipSupported()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = "http://example.com") } else { @@ -296,8 +295,8 @@ class DataUtilTest { @Test fun loadsGzipFile() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } val input: String = TestHelper.getResourceAbsolutePath("htmltests/gzip.html.gz") @@ -309,8 +308,8 @@ class DataUtilTest { @Test fun loadsZGzipFile() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } // compressed on win, with z suffix @@ -322,8 +321,8 @@ class DataUtilTest { @Test fun handlesFakeGzipFile() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } val input: String = TestHelper.getResourceAbsolutePath("htmltests/fake-gzip.html.gz") @@ -334,8 +333,8 @@ class DataUtilTest { @Test fun handlesChunkedInputStream() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } val resourceName = "htmltests/large.html.gz" diff --git a/ksoup-test/test/com/fleeksoft/ksoup/integration/ParseTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/integration/ParseTest.kt index 84bc81ec..18380c08 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/integration/ParseTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/integration/ParseTest.kt @@ -1,8 +1,9 @@ package com.fleeksoft.ksoup.integration -import com.fleeksoft.ksoup.* +import com.fleeksoft.ksoup.Ksoup import com.fleeksoft.ksoup.Ksoup.parse import com.fleeksoft.ksoup.Ksoup.parseFile +import com.fleeksoft.ksoup.TestHelper import com.fleeksoft.ksoup.nodes.Document import com.fleeksoft.ksoup.parser.Parser import com.fleeksoft.ksoup.ported.openSourceReader @@ -18,7 +19,7 @@ class ParseTest { @Test fun testHtml5Charset() = runTest { - if (Platform.isApple() || Platform.isWindows() || (BuildConfig.isKotlinx && Platform.isJsOrWasm())) { + if (!TestHelper.isGB2312Supported()) { // don't support gb2312 or gbk return@runTest } @@ -74,7 +75,7 @@ class ParseTest { @Test fun testLowercaseUtf8Charset() = runTest { val resourceName = "htmltests/lowercase-charset-test.html" - val doc: Document = if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + val doc: Document = if (!TestHelper.canReadResourceFile()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName) } else { @@ -90,7 +91,7 @@ class ParseTest { fun testXwiki() = runTest { // this tests that when in CharacterReader we hit a buffer while marked, we preserve the mark when buffered up and can rewind val resourceName = "htmltests/xwiki-1324.html.gz" - val doc: Document = if (BuildConfig.isKotlinx) { + val doc: Document = if (!TestHelper.isGzipSupported()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = "https://localhost/") } else { @@ -150,7 +151,7 @@ class ParseTest { @Test fun testFileParseNoCharsetMethod() = runTest { val resourceName = "htmltests/xwiki-1324.html.gz" - val doc: Document = if (BuildConfig.isKotlinx) { + val doc: Document = if (!TestHelper.isGzipSupported()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName) } else { diff --git a/ksoup-test/test/com/fleeksoft/ksoup/issues/GithubIssuesTests.kt b/ksoup-test/test/com/fleeksoft/ksoup/issues/GithubIssuesTests.kt index cc3d6d8c..1c425201 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/issues/GithubIssuesTests.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/issues/GithubIssuesTests.kt @@ -17,7 +17,6 @@ class GithubIssuesTests { .firstOrNull()?.let { element -> val titles = element.select("div[class=dtit]") val contents = element.select("div[class=img]") - println("titles: ${titles.size}, contents: ${contents.size}") assertEquals(6, titles.size) assertEquals(6, contents.size) } diff --git a/ksoup-test/test/com/fleeksoft/ksoup/nodes/DocumentTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/nodes/DocumentTest.kt index 8ea69772..3ae75050 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/nodes/DocumentTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/nodes/DocumentTest.kt @@ -150,7 +150,7 @@ class DocumentTest { fun testLocation() = runTest { // tests location vs base href val resourceName = "htmltests/basehref.html" - val doc: Document = if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + val doc: Document = if (!TestHelper.canReadResourceFile()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = "http://example.com/", charsetName = "UTF-8") } else { diff --git a/ksoup-test/test/com/fleeksoft/ksoup/nodes/ElementIT.kt b/ksoup-test/test/com/fleeksoft/ksoup/nodes/ElementIT.kt index 9f0d1b60..38bd6fa3 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/nodes/ElementIT.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/nodes/ElementIT.kt @@ -2,8 +2,10 @@ package com.fleeksoft.ksoup.nodes import com.fleeksoft.ksoup.* import com.fleeksoft.ksoup.Ksoup.parse -import kotlin.test.* import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue class ElementIT { @@ -139,7 +141,7 @@ class ElementIT { @Test fun wrapNoOverflow() { - if (BuildConfig.isGithubActions && Platform.isWasmJs()) { + if (Platform.isWasmJs()) { // FIXME: timeout error for js return } diff --git a/ksoup-test/test/com/fleeksoft/ksoup/nodes/NodeTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/nodes/NodeTest.kt index 896fbcbf..f8d8bd9d 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/nodes/NodeTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/nodes/NodeTest.kt @@ -2,7 +2,6 @@ package com.fleeksoft.ksoup.nodes import com.fleeksoft.ksoup.BuildConfig import com.fleeksoft.ksoup.Ksoup -import com.fleeksoft.ksoup.TestHelper import com.fleeksoft.ksoup.parser.Parser import com.fleeksoft.ksoup.parser.Tag import com.fleeksoft.ksoup.select.NodeVisitor @@ -33,10 +32,10 @@ class NodeTest { assertEquals("", withBase.absUrl("noval")) val dodgyBase = Element(tag, "wtf://no-such-protocol/", attribs) assertEquals("http://bar/qux", dodgyBase.absUrl("absHref")) // base fails, but href good, so get that - if (BuildConfig.isKotlinx) { - assertEquals("", dodgyBase.absUrl("relHref")) // base fails, only rel href, so return nothing - } else { + if (BuildConfig.isKorlibs) { assertEquals("wtf://no-such-protocol/foo", dodgyBase.absUrl("relHref")) // invalid protocol but still can be resolved + } else { + assertEquals("", dodgyBase.absUrl("relHref")) // base fails, only rel href, so return nothing } } @@ -96,11 +95,11 @@ class NodeTest { val one = doc.select("a").first() assertEquals("file:///etc/password", one!!.absUrl("href")) val two = doc.select("a")[1] - if (BuildConfig.isKotlinx) { -// fixme: in kotlinx its different behavoiru - assertEquals("file://var/log/messages", two.absUrl("href")) - } else { + if (BuildConfig.isKorlibs) { assertEquals("file:///var/log/messages", two.absUrl("href")) + } else { + // fixme: in kotlinx its different behaviour + assertEquals("file://var/log/messages", two.absUrl("href")) } } diff --git a/ksoup-test/test/com/fleeksoft/ksoup/parser/CharacterReaderTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/parser/CharacterReaderTest.kt index f6e2ec4a..31365aff 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/parser/CharacterReaderTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/parser/CharacterReaderTest.kt @@ -1,10 +1,7 @@ package com.fleeksoft.ksoup.parser -import com.fleeksoft.ksoup.BuildConfig -import com.fleeksoft.ksoup.Platform import com.fleeksoft.ksoup.TestHelper import com.fleeksoft.ksoup.internal.StringUtil -import com.fleeksoft.ksoup.isJsOrWasm import com.fleeksoft.ksoup.ported.exception.UncheckedIOException import com.fleeksoft.ksoup.ported.io.Charsets import com.fleeksoft.ksoup.ported.io.StringReader @@ -23,7 +20,7 @@ class CharacterReaderTest { @Test fun testUtf16BE() = runTest { - if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + if (!TestHelper.isUtf16Supported()) { // not supported in kotlinx for js return@runTest } @@ -38,7 +35,7 @@ class CharacterReaderTest { @Test fun testUtf16LE() = runTest { - if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + if (!TestHelper.isUtf16Supported()) { // not supported in kotlinx for js return@runTest } diff --git a/ksoup-test/test/com/fleeksoft/ksoup/parser/HtmlParserTest.kt b/ksoup-test/test/com/fleeksoft/ksoup/parser/HtmlParserTest.kt index aeeda564..20289c25 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/parser/HtmlParserTest.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/parser/HtmlParserTest.kt @@ -1290,7 +1290,7 @@ class HtmlParserTest { @Test fun testInvalidTableContents() = runTest { val resourceName = "htmltests/table-invalid-elements.html" - val doc: Document = if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + val doc: Document = if (!TestHelper.canReadResourceFile()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName, charsetName = "UTF-8") } else { @@ -1507,7 +1507,7 @@ class HtmlParserTest { @Test fun testTemplateInsideTable() = runTest { val resourceName = "htmltests/table-polymer-template.html" - val doc: Document = if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + val doc: Document = if (!TestHelper.canReadResourceFile()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName, charsetName = "UTF-8") } else { @@ -1550,7 +1550,7 @@ class HtmlParserTest { @Test fun handlesXmlDeclAndCommentsBeforeDoctype() = runTest { val resourceName = "htmltests/comments.html" - val doc: Document = if (BuildConfig.isKotlinx && Platform.isJsOrWasm()) { + val doc: Document = if (!TestHelper.canReadResourceFile()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName, charsetName = "UTF-8") } else { @@ -1583,7 +1583,7 @@ class HtmlParserTest { @Test fun characterReaderBuffer() = runTest { val resourceName = "htmltests/character-reader-buffer.html.gz" - val doc: Document = if (BuildConfig.isKotlinx) { + val doc: Document = if (!TestHelper.canReadResourceFile() || !TestHelper.isGzipSupported()) { val source = TestHelper.readResource(resourceName) Ksoup.parse(sourceReader = source, baseUri = resourceName, charsetName = "UTF-8") } else { diff --git a/ksoup-test/test/com/fleeksoft/ksoup/parser/ParserIT.kt b/ksoup-test/test/com/fleeksoft/ksoup/parser/ParserIT.kt index 2aab36fc..1f0ff571 100644 --- a/ksoup-test/test/com/fleeksoft/ksoup/parser/ParserIT.kt +++ b/ksoup-test/test/com/fleeksoft/ksoup/parser/ParserIT.kt @@ -1,7 +1,13 @@ package com.fleeksoft.ksoup.parser -import com.fleeksoft.ksoup.* -import kotlin.test.* +import com.fleeksoft.ksoup.BuildConfig +import com.fleeksoft.ksoup.Platform +import com.fleeksoft.ksoup.System +import com.fleeksoft.ksoup.isJsOrWasm +import kotlin.test.Ignore +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue /** * Longer running Parser tests. @@ -48,7 +54,11 @@ class ParserIT { // Assert assertEquals(2, doc.body().childNodeSize()) assertEquals(25000, doc.select("dd").size) - assertTrue(System.currentTimeMillis() - start < 40000) // I get ~ 1.5 seconds, but others have reported slower + val end = System.currentTimeMillis() - start + assertTrue( + end < 40000, + "Expected max time for this test was: 40 secs but it took ${end.div(1000)}" + )// I get ~ 1.5 seconds, but others have reported slower // was originally much longer, or stack overflow. } } diff --git a/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/DataUtilTestJvm.kt b/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/DataUtilTestJvm.kt index b93eca99..4f653609 100644 --- a/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/DataUtilTestJvm.kt +++ b/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/DataUtilTestJvm.kt @@ -24,8 +24,8 @@ class DataUtilTestJvm { @Test fun loadsGzipPath() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } val `in`: Path = ParserHelper.getPath("/htmltests/gzip.html.gz") @@ -36,8 +36,8 @@ class DataUtilTestJvm { @Test fun loadsZGzipPath() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } // compressed on win, with z suffix @@ -49,8 +49,8 @@ class DataUtilTestJvm { @Test fun handlesFakeGzipPath() = runTest { - if (BuildConfig.isKotlinx) { -// kotlinx module not support gzip + if (!TestHelper.isGzipSupported()) { +// gzip not supported for this return@runTest } val `in`: Path = ParserHelper.getPath("htmltests/fake-gzip.html.gz") diff --git a/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/io/InputStreamReader.test.kt b/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/io/InputStreamReader.test.kt index 7224557f..936d38d4 100644 --- a/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/io/InputStreamReader.test.kt +++ b/ksoup-test/test@jvmAndAndroid/com/fleeksoft/ksoup/io/InputStreamReader.test.kt @@ -1,6 +1,5 @@ package com.fleeksoft.ksoup.io -import com.fleeksoft.ksoup.JvmKotlinxMapper import com.fleeksoft.ksoup.internal.SharedConstants import com.fleeksoft.ksoup.ported.io.BufferedReader import com.fleeksoft.ksoup.ported.io.InputSourceReader @@ -14,7 +13,7 @@ import kotlin.test.assertEquals class InputStreamReader { private fun String.toInputStreamSourceReader(): SourceReader { - return JvmKotlinxMapper.jvmInputStreamToSourceReader(ByteArrayInputStream(this.encodeToByteArray())) + return SourceReader.from(ByteArrayInputStream(this.encodeToByteArray())) } @Test diff --git a/ksoup/build.gradle.kts b/ksoup/build.gradle.kts index d8229081..256302ef 100644 --- a/ksoup/build.gradle.kts +++ b/ksoup/build.gradle.kts @@ -11,18 +11,29 @@ kotlin { sourceSets { commonMain { dependencies { - println("libBuildType: $libBuildType") - if (libBuildType == "korlibs") { - api(project(":ksoup-engine-korlibs")) - } else { - api(project(":ksoup-engine-kotlinx")) + when (libBuildType) { + "korlibs" -> { + api(project(":ksoup-engine-korlibs")) + } + + "okio" -> { + api(project(":ksoup-engine-okio")) + } + + "ktor2" -> { + api(project(":ksoup-engine-ktor2")) + } + + else -> { + api(project(":ksoup-engine-kotlinx")) + } } } } } } -val artifactId = if (libBuildType == "korlibs") "ksoup-korlibs" else "ksoup" +val artifactId = if (libBuildType == "korlibs") "ksoup-korlibs" else if (libBuildType == "okio") "ksoup-okio" else "ksoup" mavenPublishing { coordinates("com.fleeksoft.ksoup", artifactId, libs.versions.libraryVersion.get()) pom { diff --git a/ksoup/module.yaml b/ksoup/module.yaml index 139f365f..3758dc60 100644 --- a/ksoup/module.yaml +++ b/ksoup/module.yaml @@ -1,6 +1,6 @@ product: type: lib - platforms: [jvm, js, wasm, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64] + platforms: [jvm, js, android, linuxX64, linuxArm64, tvosArm64, tvosX64, tvosSimulatorArm64, macosX64, macosArm64, iosArm64, iosSimulatorArm64, iosX64, mingwX64] apply: [ ../common.module-template.yaml ] diff --git a/ksoup/src@jvmAndAndroid/com/fleeksoft/ksoup/KsoupJvmExt.kt b/ksoup/src@jvmAndAndroid/com/fleeksoft/ksoup/KsoupJvmExt.kt index b532887c..35bd5d79 100644 --- a/ksoup/src@jvmAndAndroid/com/fleeksoft/ksoup/KsoupJvmExt.kt +++ b/ksoup/src@jvmAndAndroid/com/fleeksoft/ksoup/KsoupJvmExt.kt @@ -1,7 +1,11 @@ package com.fleeksoft.ksoup +import com.fleeksoft.ksoup.io.FileSource +import com.fleeksoft.ksoup.io.SourceReader +import com.fleeksoft.ksoup.io.from import com.fleeksoft.ksoup.nodes.Document import com.fleeksoft.ksoup.parser.Parser +import com.fleeksoft.ksoup.ported.toSourceFile import java.io.File import java.io.IOException import java.io.InputStream @@ -26,7 +30,7 @@ public fun Ksoup.parseInputStream( parser: Parser = Parser.htmlParser(), ): Document { return parse( - sourceReader = JvmKotlinxMapper.jvmInputStreamToSourceReader(inputStream), + sourceReader = SourceReader.from(inputStream), charsetName = charsetName, baseUri = baseUri, parser = parser, @@ -50,7 +54,7 @@ public suspend fun Ksoup.parseFile( parser: Parser = Parser.htmlParser(), ): Document { return parseFile( - file = JvmKotlinxMapper.jvmFileToFileSource(file), + file = FileSource.from(file), charsetName = charsetName, baseUri = baseUri, parser = parser, @@ -77,7 +81,7 @@ suspend fun Ksoup.parsePath( parser: Parser = Parser.htmlParser() ): Document { return Ksoup.parse( - sourceReader = KsoupEngineInstance.ksoupEngine.pathToFileSource(path.absolutePathString()).toSourceReader(), + sourceReader = path.absolutePathString().toSourceFile().toSourceReader(), baseUri = baseUri, charsetName = charsetName, parser = parser diff --git a/publishToLocal.sh b/publishToLocal.sh deleted file mode 100755 index 2a1e72e0..00000000 --- a/publishToLocal.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Stop the script if any command fails -set -e - -./gradlew clean -./gradlew :ksoup-engine-common:publishToMavenLocal -./gradlew :ksoup-engine-kotlinx:publishToMavenLocal -./gradlew :ksoup-engine-korlibs:publishToMavenLocal - -./gradlew clean -./gradlew :ksoup:publishToMavenLocal -PlibBuildType=kotlinx -./gradlew :ksoup-network:publishToMavenLocal -PlibBuildType=kotlinx - -./gradlew clean -./gradlew :ksoup:publishToMavenLocal -PlibBuildType=korlibs -./gradlew :ksoup-network-korlibs:publishToMavenLocal -PlibBuildType=korlibs - -echo "Publishing completed successfully." \ No newline at end of file diff --git a/publishToMaven.sh b/publishToMaven.sh index a700868c..a0c046f5 100755 --- a/publishToMaven.sh +++ b/publishToMaven.sh @@ -3,19 +3,101 @@ # Stop the script if any command fails set -e -./gradlew clean -./gradlew :ksoup-engine-common:publishAllPublicationsToMavenCentralRepository -./gradlew clean -./gradlew :ksoup-engine-kotlinx:publishAllPublicationsToMavenCentralRepository -PlibBuildType=kotlinx -./gradlew clean -./gradlew :ksoup-engine-korlibs:publishAllPublicationsToMavenCentralRepository -PlibBuildType=korlibs - -./gradlew clean -./gradlew :ksoup:publishAllPublicationsToMavenCentralRepository -PlibBuildType=korlibs -./gradlew :ksoup-network-korlibs:publishAllPublicationsToMavenCentralRepository -PlibBuildType=korlibs - -./gradlew clean -./gradlew :ksoup-network:publishAllPublicationsToMavenCentralRepository -PlibBuildType=kotlinx -./gradlew :ksoup:publishAllPublicationsToMavenCentralRepository -PlibBuildType=kotlinx - -echo "Publishing completed successfully." \ No newline at end of file +# Flag to determine whether to add wasm or not +ADD_WASM=true + +# Default publishing task +PUBLISH_TASK="publishToMavenLocal" + +# Check for the --remote flag +if [ "$1" == "--remote" ]; then + PUBLISH_TASK="publishAllPublicationsToMavenCentralRepository" +fi + +# projectModule:libBuildType +projects=( + "ksoup-engine-common:" + "ksoup-engine-kotlinx:" + "ksoup-engine-korlibs:" + "ksoup-engine-ktor2:" + "ksoup-engine-okio:" + "ksoup:kotlinx" + "ksoup-network:kotlinx" + "ksoup:korlibs" + "ksoup-network-korlibs:korlibs" + "ksoup:ktor2" + "ksoup-network-ktor2:ktor2" + "ksoup:okio" +) + +# Function to add wasm to platforms list if not already present +add_wasm_platform() { + local module_file="$1/module.yaml" + + # Check if 'platforms:' line already contains 'wasm' + if grep -q 'platforms: \[.*wasm' "$module_file"; then + echo "wasm is already in the platforms list in $module_file" + return 0 # Return 0 (indicating wasm was not added) + else + echo "Adding wasm to platforms list in $module_file" + cp "$module_file" "$module_file.bak" + # Add 'wasm' to the beginning of the platforms list + sed -i.bak 's/\(platforms: \[\)/\1wasm, /' "$module_file" + return 1 # Return 1 (indicating wasm was added) + fi +} + +# Function to restore the original module.yaml file +restore_module_yaml() { + echo "Restoring original module.yaml in $1" + mv "$1/module.yaml.bak" "$1/module.yaml" +} + +# Function to handle errors and restore the original file if needed +error_handler() { + echo "Error detected, check for restore of module.yaml.bak if needed for $projectName" + if [ -f "$projectName/module.yaml.bak" ]; then + restore_module_yaml "$projectName" + fi + exit 1 +} + +# Set trap to catch any errors and call the error_handler function +trap 'error_handler' ERR + +# Loop through all projects and publish them +for project in "${projects[@]}"; do + # Split the project name and build type + IFS=":" read -r projectName buildType <<< "$project" + + wasm_added=0 + + if [ "$ADD_WASM" = true ] && [[ "$buildType" == "kotlinx" || "$buildType" == "korlibs" ]]; then + trap - ERR # Temporarily disable the trap + set +e # Disable exit on error + # Add wasm to platforms list if buildType is kotlinx or korlibs + add_wasm_platform "$projectName" + wasm_added=$? # Capture the return value indicate if wasm added + set -e # Re-enable exit on error + trap 'error_handler' ERR # Re-enable the trap + else + wasm_added=0 # Set to false if wasm wasn't added + fi + + if [ -n "$buildType" ]; then + ./gradlew clean -PlibBuildType="$buildType" --quiet --warning-mode=none + echo "Publishing $projectName with libBuildType=$buildType" + ./gradlew ":$projectName:$PUBLISH_TASK" -PlibBuildType="$buildType" --quiet --warning-mode=none + else + ./gradlew clean --quiet --warning-mode=none + echo "Publishing $projectName" + ./gradlew ":$projectName:$PUBLISH_TASK" --quiet --warning-mode=none + fi + + # Restore the original module.yaml file after publishing if wasm was added + if [ "$wasm_added" -eq 1 ]; then + restore_module_yaml "$projectName" + fi +done + +echo "Publishing completed successfully." diff --git a/runAllTests.sh b/runAllTests.sh deleted file mode 100755 index b5b00453..00000000 --- a/runAllTests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Stop the script if any command fails -set -e - -# Function to run tests for a specific configuration -run_tests() { - local libBuildType=$1 - - echo "Running tests with libBuildType=$libBuildType..." - - ./gradlew clean - ./gradlew jvmTest testDebugUnitTest testReleaseUnitTest -PlibBuildType="$libBuildType" - rm -rf kotlin-js-store - ./gradlew jsTest wasmTest -PlibBuildType="$libBuildType" - ./gradlew iosX64Test iosSimulatorArm64Test macosX64Test macosArm64Test tvosX64Test tvosSimulatorArm64Test -PlibBuildType="$libBuildType" -} - -# Run tests for kotlinx -run_tests kotlinx - -# Run tests for korlibs -run_tests korlibs - -echo "All tests run successfully!" \ No newline at end of file diff --git a/runTests.sh b/runTests.sh new file mode 100755 index 00000000..cdd1847c --- /dev/null +++ b/runTests.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Stop the script if any command fails +set -e + +# Function to add wasm to platforms list if not already present +add_wasm_platform() { + local module_file="ksoup/module.yaml" + + if grep -q 'platforms: \[.*wasm' "$module_file"; then + echo "wasm is already in the platforms list in $module_file" + return 0 + else + echo "Adding wasm to platforms list in $module_file" + cp "$module_file" "$module_file.bak" + sed -i.bak 's/\(platforms: \[\)/\1wasm, /' "$module_file" + return 1 + fi +} + +# Function to restore the original module.yaml file +restore_module_yaml() { + echo "Restoring original module.yaml..." + mv "ksoup/module.yaml.bak" "ksoup/module.yaml" +} + +# Error handler function to restore module.yaml on failure +error_handler() { + echo "Error detected, restoring module.yaml if necessary..." + if [ -f "ksoup/module.yaml.bak" ]; then + restore_module_yaml + fi + exit 1 +} + +# Function to safely remove a directory if it exists +safe_remove_dir() { + local dir="$1" + if [ -d "$dir" ]; then + rm -rf "$dir" + fi +} + +# Set trap to catch any errors and call the error_handler function +trap 'error_handler' ERR + +# Function to run tests for a specific configuration +run_tests() { + local libBuildType="$1" + shift + local tasks=("$@") + + if [ ${#tasks[@]} -eq 0 ]; then + echo "No specific tasks provided, running all default tests..." + tasks=("jvmTest" "testDebugUnitTest" "testReleaseUnitTest" "jsTest" "wasmTest" "iosX64Test" "iosSimulatorArm64Test" "macosX64Test" "macosArm64Test" "tvosX64Test" "tvosSimulatorArm64Test") + fi + + echo "Running tests with libBuildType=$libBuildType and tasks=${tasks[*]}..." + + # Only add/remove wasm for kotlinx and korlibs + local wasm_added=0 + if [[ "$libBuildType" == "kotlinx" || "$libBuildType" == "korlibs" ]]; then + trap - ERR # Temporarily disable the trap + set +e # Disable exit on error + add_wasm_platform + wasm_added=$? + set -e # Re-enable exit on error + trap 'error_handler' ERR # Re-enable the trap + fi + + # Remove build directories if they exist + echo "remove build dirs if exists" + safe_remove_dir ".kotlin" + safe_remove_dir "build" + safe_remove_dir ".gradle" + + ./gradlew clean -PlibBuildType="$libBuildType" --quiet --warning-mode=none + + for task in "${tasks[@]}"; do + safe_remove_dir "kotlin-js-store" #remove it every task to avoid lock issue + start_time=$(date +%s) + echo "Running $task... $libBuildType" + ./gradlew "$task" -PlibBuildType="$libBuildType" --quiet --warning-mode=none + end_time=$(date +%s) + duration=$((end_time - start_time)) + echo "Task $task completed in $duration seconds." + done + + # Restore original module.yaml if wasm was added + if [ "$wasm_added" -eq 1 ]; then + restore_module_yaml + fi +} + +# Supported parameters +SUPPORTED_PARAMS=("korlibs" "okio" "kotlinx" "ktor2") + +# Function to check if the provided parameter is supported +is_supported_param() { + local param="$1" + for supported_param in "${SUPPORTED_PARAMS[@]}"; do + if [ "$supported_param" == "$param" ]; then + return 0 + fi + done + return 1 +} + +# Main script logic +if [ "$#" -ge 1 ]; then + libBuildType="$1" + shift + + if is_supported_param "$libBuildType"; then + run_tests "$libBuildType" "$@" + else + echo "Error: Unsupported parameter '$libBuildType'. Supported parameters are: ${SUPPORTED_PARAMS[*]}" + exit 1 + fi +else + for param in "${SUPPORTED_PARAMS[@]}"; do + run_tests "$param" + done +fi + +echo "All tests ran successfully!" diff --git a/settings.gradle.kts b/settings.gradle.kts index 42c664b8..bd5b1813 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,7 +10,7 @@ pluginManagement { } plugins { - id("org.jetbrains.amper.settings.plugin").version("0.5.0-dev-973") + id("org.jetbrains.amper.settings.plugin").version("0.5.0-dev-992") } val libBuildType = settings.providers.gradleProperty("libBuildType").get() @@ -21,10 +21,19 @@ if (libBuildType == "korlibs" || libBuildType == "dev") { } if (libBuildType == "kotlinx" || libBuildType == "dev") { - println("add engine kotlinx") include("ksoup-engine-kotlinx", "ksoup-network") } + +if (libBuildType == "okio" || libBuildType == "dev") { + include("ksoup-engine-okio", "ksoup-network-ktor2") +} + +if (libBuildType == "ktor2" || libBuildType == "dev") { + include("ksoup-engine-ktor2", "ksoup-network-ktor2") +} + include("ksoup") include("ksoup-test") + //include("sample:shared", "sample:desktop") //include("sample:android", "sample:ios") \ No newline at end of file