diff --git a/src/androidInstrumentedTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.android.kt b/src/androidInstrumentedTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.android.kt index d192861..bba0b07 100644 --- a/src/androidInstrumentedTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.android.kt +++ b/src/androidInstrumentedTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.android.kt @@ -1,3 +1,5 @@ +@file:Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") + package com.github.lamba92.leveldb.tests import androidx.test.platform.app.InstrumentationRegistry @@ -10,3 +12,5 @@ actual val DATABASE_PATH: String .filesDir .resolve("testdb") .absolutePath + +actual typealias Test = org.junit.Test diff --git a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/LevelDBUtils.jvm.kt b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/LevelDBUtils.jvm.kt index b7fbf0e..9732f4f 100644 --- a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/LevelDBUtils.jvm.kt +++ b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/LevelDBUtils.jvm.kt @@ -4,10 +4,26 @@ import com.github.lamba92.leveldb.jvm.JvmLevelDB import com.github.lamba92.leveldb.jvm.LibLevelDB import com.github.lamba92.leveldb.jvm.toNative import com.sun.jna.ptr.PointerByReference +import java.nio.charset.Charset public actual fun LevelDB( path: String, options: LevelDBOptions, +): LevelDB = LevelDB(path, options, Charsets.UTF_8) + +/** + * Creates a new instance of a LevelDB database at the specified file path. + * + * @param path The file system path where the LevelDB database is located or will be created. It has to a directory. + * If the directory does not exist, it will be created only if [LevelDBOptions.createIfMissing] is set to `true`. + * @param options Configuration options for creating and managing the LevelDB instance, defaults to [LevelDBOptions.DEFAULT]. + * @param charset The character set to use for encoding and decoding strings. Defaults to [Charsets.UTF_8]. + * @return A `LevelDB` instance for interacting with the database. + */ +public fun LevelDB( + path: String, + options: LevelDBOptions, + charset: Charset, ): LevelDB { val nativeOptions = options.toNative() val errPtr = PointerByReference() @@ -17,7 +33,7 @@ public actual fun LevelDB( LibLevelDB.leveldb_free(errPtr.value) error("Failed to open database: $errorValue") } - return JvmLevelDB(nativeDelegate, nativeOptions) + return JvmLevelDB(nativeDelegate, nativeOptions, charset) } public actual fun repairDatabase( diff --git a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDB.kt b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDB.kt index b639d9b..ea5c83d 100644 --- a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDB.kt +++ b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDB.kt @@ -7,10 +7,12 @@ import com.github.lamba92.leveldb.LevelDBBatchOperation import com.github.lamba92.leveldb.LevelDBReader import com.github.lamba92.leveldb.LevelDBSnapshot import com.sun.jna.ptr.PointerByReference +import java.nio.charset.Charset public class JvmLevelDB internal constructor( private val nativeDatabase: LibLevelDB.leveldb_t, private val nativeOptions: LibLevelDB.leveldb_options_t, + private val charset: Charset, ) : LevelDB { override fun put( key: String, @@ -21,15 +23,15 @@ public class JvmLevelDB internal constructor( val errPtr = PointerByReference() val writeOptions = leveldb_writeoptions_create() leveldb_writeoptions_set_sync(writeOptions, sync.toByte()) - val keyPointer = key.toPointer() - val valuePointer = value.toPointer() + val keyPointer = key.toPointer(charset) + val valuePointer = value.toPointer(charset) leveldb_put( db = nativeDatabase, options = writeOptions, key = keyPointer, - keylen = key.length.toNativeLong(), + keylen = keyPointer.size().toNativeLong(), value = valuePointer, - vallen = value.length.toNativeLong(), + vallen = valuePointer.size().toNativeLong(), errptr = errPtr, ) valuePointer.close() @@ -46,7 +48,7 @@ public class JvmLevelDB internal constructor( key: String, verifyChecksums: Boolean, fillCache: Boolean, - ): String? = nativeDatabase.get(verifyChecksums, fillCache, key) + ): String? = nativeDatabase.get(verifyChecksums, fillCache, key, charset = charset) override fun delete( key: String, @@ -56,12 +58,12 @@ public class JvmLevelDB internal constructor( val errPtr = PointerByReference() val writeOptions = leveldb_writeoptions_create() leveldb_writeoptions_set_sync(writeOptions, sync.toByte()) - val keyPointer = key.toPointer() + val keyPointer = key.toPointer(charset) leveldb_delete( db = nativeDatabase, options = writeOptions, key = keyPointer, - keylen = key.length.toNativeLong(), + keylen = keyPointer.size().toNativeLong(), errptr = errPtr, ) keyPointer.clear(key.length.toLong()) @@ -83,25 +85,25 @@ public class JvmLevelDB internal constructor( for (operation in operations) { when (operation) { is LevelDBBatchOperation.Put -> { - val keyPointer = operation.key.toPointer() - val valuePointer = operation.value.toPointer() + val keyPointer = operation.key.toPointer(charset) + val valuePointer = operation.value.toPointer(charset) leveldb_writebatch_put( batch = nativeBatch, key = keyPointer, - klen = operation.key.length.toNativeLong(), + klen = keyPointer.size().toNativeLong(), value = valuePointer, - vlen = operation.value.length.toNativeLong(), + vlen = valuePointer.size().toNativeLong(), ) keyPointer.close() valuePointer.close() } is LevelDBBatchOperation.Delete -> { - val keyPointer = operation.key.toPointer() + val keyPointer = operation.key.toPointer(charset) leveldb_writebatch_delete( batch = nativeBatch, key = keyPointer, - klen = operation.key.length.toNativeLong(), + klen = keyPointer.size().toNativeLong(), ) keyPointer.close() } @@ -128,13 +130,19 @@ public class JvmLevelDB internal constructor( from: String?, verifyChecksums: Boolean, fillCache: Boolean, - ): CloseableSequence = nativeDatabase.asSequence(verifyChecksums, fillCache, from) + ): CloseableSequence = + nativeDatabase.asSequence( + verifyChecksums, + fillCache, + from, + charset = charset, + ) @BrokenNativeAPI override fun withSnapshot(action: LevelDBSnapshot.() -> T): T { val nativeSnapshot = LibLevelDB.leveldb_create_snapshot(nativeDatabase) return try { - action(JvmLevelDBSnapshot(nativeDatabase, nativeSnapshot)) + action(JvmLevelDBSnapshot(nativeDatabase, nativeSnapshot, charset)) } finally { LibLevelDB.leveldb_release_snapshot(nativeDatabase, nativeSnapshot) } @@ -147,11 +155,11 @@ public class JvmLevelDB internal constructor( val startPointer = start .takeIf { it.isNotEmpty() } - ?.toPointer() + ?.toPointer(charset) val endPointer = end .takeIf { it.isNotEmpty() } - ?.toPointer() + ?.toPointer(charset) LibLevelDB.leveldb_compact_range( db = nativeDatabase, start_key = startPointer, diff --git a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDBSnapshot.kt b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDBSnapshot.kt index c621977..ba1f065 100644 --- a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDBSnapshot.kt +++ b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmLevelDBSnapshot.kt @@ -5,10 +5,12 @@ import com.github.lamba92.leveldb.LevelDBReader import com.github.lamba92.leveldb.LevelDBSnapshot import kotlinx.datetime.Clock import kotlinx.datetime.Instant +import java.nio.charset.Charset public class JvmLevelDBSnapshot internal constructor( private val nativeDatabase: LibLevelDB.leveldb_t, private val nativeSnapshot: LibLevelDB.leveldb_snapshot_t, + private val charset: Charset, ) : LevelDBSnapshot { override val createdAt: Instant = Clock.System.now() @@ -16,11 +18,25 @@ public class JvmLevelDBSnapshot internal constructor( key: String, verifyChecksums: Boolean, fillCache: Boolean, - ): String? = nativeDatabase.get(verifyChecksums, fillCache, key, nativeSnapshot) + ): String? = + nativeDatabase.get( + verifyChecksums = verifyChecksums, + fillCache = fillCache, + key = key, + snapshot = nativeSnapshot, + charset = charset, + ) override fun scan( from: String?, verifyChecksums: Boolean, fillCache: Boolean, - ): CloseableSequence = nativeDatabase.asSequence(verifyChecksums, fillCache, from, nativeSnapshot) + ): CloseableSequence = + nativeDatabase.asSequence( + verifyChecksums = verifyChecksums, + fillCache = fillCache, + from = from, + snapshot = nativeSnapshot, + charset = charset, + ) } diff --git a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmUtills.kt b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmUtills.kt index d23734e..4b7c140 100644 --- a/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmUtills.kt +++ b/src/jvmCommonMain/kotlin/com/github/lamba92/leveldb/jvm/JvmUtills.kt @@ -6,16 +6,24 @@ import com.github.lamba92.leveldb.LevelDBReader import com.github.lamba92.leveldb.asCloseable import com.sun.jna.Memory import com.sun.jna.NativeLong +import com.sun.jna.Pointer import com.sun.jna.ptr.LongByReference import com.sun.jna.ptr.PointerByReference +import java.nio.charset.Charset -internal fun String.toPointer() = Memory(length.toLong() + 1L).also { it.setString(0L, this) } +internal fun String.toPointer(charset: Charset): Memory { + val data = toByteArray(charset) + val mem = Memory(data.size.toLong()) + mem.write(0, data, 0, data.size) + return mem +} internal fun LibLevelDB.leveldb_t.get( verifyChecksums: Boolean, fillCache: Boolean, key: String, snapshot: LibLevelDB.leveldb_snapshot_t? = null, + charset: Charset, ) = with(LibLevelDB) { val errPtr = PointerByReference() val nativeReadOptions = leveldb_readoptions_create() @@ -24,14 +32,14 @@ internal fun LibLevelDB.leveldb_t.get( if (snapshot != null) { leveldb_readoptions_set_snapshot(nativeReadOptions, snapshot) } - val keyPointer = key.toPointer() + val keyPointer = key.toPointer(charset) val valueLengthPointer = LongByReference() val value = leveldb_get( db = this@get, options = nativeReadOptions, key = keyPointer, - keylen = key.length.toNativeLong(), + keylen = keyPointer.size().toNativeLong(), vallen = valueLengthPointer, errptr = errPtr, ) @@ -42,8 +50,7 @@ internal fun LibLevelDB.leveldb_t.get( if (errorValue != null) { error("Failed to get value: $errorValue") } - value?.getByteArray(0, valueLength.toInt()) - ?.toString(Charsets.UTF_8) + value?.toString(valueLength.toInt(), charset) } internal fun LevelDBOptions.toNative() = @@ -71,6 +78,7 @@ internal fun LibLevelDB.leveldb_t.asSequence( fillCache: Boolean, from: String? = null, snapshot: LibLevelDB.leveldb_snapshot_t? = null, + charset: Charset, ): CloseableSequence = with(LibLevelDB) { val nativeOptions = leveldb_readoptions_create() @@ -85,7 +93,7 @@ internal fun LibLevelDB.leveldb_t.asSequence( when (from) { null -> leveldb_iter_seek_to_first(iterator) else -> { - val fromPointer = from.toPointer() + val fromPointer = from.toPointer(charset) leveldb_iter_seek(iterator, fromPointer, from.length.toNativeLong()) fromPointer.close() } @@ -99,16 +107,12 @@ internal fun LibLevelDB.leveldb_t.asSequence( val key = lazy { val keyPointer = leveldb_iter_key(iterator, keyLengthPointer) - keyPointer.getByteArray(0, keyLengthPointer.value.toInt()) - ?.toString(Charsets.UTF_8) - ?: error("Failed to read key") + keyPointer.toString(keyLengthPointer.value.toInt(), charset) } val value = lazy { val valuePointer = leveldb_iter_value(iterator, valueLengthPointer) - valuePointer.getByteArray(0, valueLengthPointer.value.toInt()) - ?.toString(Charsets.UTF_8) - ?: error("Failed to read value for key '${key.value}'") + valuePointer.toString(valueLengthPointer.value.toInt(), charset) } yield(LevelDBReader.LazyEntry(key, value)) leveldb_iter_next(iterator) @@ -121,6 +125,14 @@ internal fun LibLevelDB.leveldb_t.asSequence( } } +private fun Pointer.toString( + length: Int, + charset: Charset, +): String = + getByteArray(0, length) + ?.toString(charset) + ?: error("Failed to read string") + internal fun Boolean.toByte(): Byte = if (this) 1 else 0 internal fun Number.toNativeLong() = NativeLong(toLong()) diff --git a/src/jvmCommonTest/kotlin/com/github/lamba92/leveldb/tests/UTF16Tests.kt b/src/jvmCommonTest/kotlin/com/github/lamba92/leveldb/tests/UTF16Tests.kt new file mode 100644 index 0000000..eb055db --- /dev/null +++ b/src/jvmCommonTest/kotlin/com/github/lamba92/leveldb/tests/UTF16Tests.kt @@ -0,0 +1,73 @@ +@file:Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING", "unused") + +package com.github.lamba92.leveldb.tests + +import kotlin.test.assertEquals + +@Target(AnnotationTarget.FUNCTION) +expect annotation class Test() + +class UTF16Tests { + @Test + fun emoji() = + withDatabase { db -> + val key = "👋🌍" + val value = "👋🌍" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun accentedChars() = + withDatabase { db -> + val key = "áéíóú" + val value = "áéíóú" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun chineseChars() = + withDatabase { db -> + val key = "你好" + val value = "你好" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun japaneseChars() = + withDatabase { db -> + val key = "こんにちは" + val value = "こんにちは" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun cyrillicChars() = + withDatabase { db -> + val key = "Здравствуйте" + val value = "Здравствуйте" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun dieresis() = + withDatabase { db -> + val key = "äëïöü" + val value = "äëïöü" + db.put(key, value) + assertEquals(value, db.get(key)) + } + + @Test + fun greekChars() = + withDatabase { db -> + val key = "Γειά σας" + val value = "Γειά σας" + db.put(key, value) + assertEquals(value, db.get(key)) + } +} diff --git a/src/jvmTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.jvm.kt b/src/jvmTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.jvm.kt index 8d166ae..c5fccfb 100644 --- a/src/jvmTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.jvm.kt +++ b/src/jvmTest/kotlin/com/github/lamba92/leveldb/tests/ApiTests.jvm.kt @@ -1,6 +1,10 @@ +@file:Suppress("EXPECT_ACTUAL_CLASSIFIERS_ARE_IN_BETA_WARNING") + package com.github.lamba92.leveldb.tests actual val DATABASE_PATH: String get() = System.getenv("LEVELDB_LOCATION") ?: error("LEVELDB_LOCATION environment variable not set") + +actual typealias Test = org.junit.jupiter.api.Test