Support incremental hashing via HashFunction

whyoleg · Sep 8, 2024 · 9a2e803 · 9a2e803
1 parent ad01bc0
commit 9a2e803
Show file tree

Hide file tree

Showing 17 changed files with 566 additions and 103 deletions.
diff --git a/cryptography-core/api/cryptography-core.api b/cryptography-core/api/cryptography-core.api
@@ -618,6 +618,22 @@ public final class dev/whyoleg/cryptography/algorithms/symmetric/SymmetricKeySiz
 	public final fun getB256-5xWg6fk ()I
 }
 
+public abstract interface class dev/whyoleg/cryptography/functions/HashFunction : dev/whyoleg/cryptography/functions/UpdateFunction {
+	public abstract fun hash ()Lkotlinx/io/bytestring/ByteString;
+	public abstract fun hashInto ([BI)I
+	public static synthetic fun hashInto$default (Ldev/whyoleg/cryptography/functions/HashFunction;[BIILjava/lang/Object;)I
+	public abstract fun reset ()V
+}
+
+public abstract interface class dev/whyoleg/cryptography/functions/UpdateFunction : java/lang/AutoCloseable {
+	public fun update (Lkotlinx/io/bytestring/ByteString;II)V
+	public abstract fun update ([BII)V
+	public static synthetic fun update$default (Ldev/whyoleg/cryptography/functions/UpdateFunction;Lkotlinx/io/bytestring/ByteString;IIILjava/lang/Object;)V
+	public static synthetic fun update$default (Ldev/whyoleg/cryptography/functions/UpdateFunction;[BIIILjava/lang/Object;)V
+	public fun updatingSink (Lkotlinx/io/RawSink;)Lkotlinx/io/RawSink;
+	public fun updatingSource (Lkotlinx/io/RawSource;)Lkotlinx/io/RawSource;
+}
+
 public abstract interface class dev/whyoleg/cryptography/materials/key/EncodableKey : dev/whyoleg/cryptography/materials/key/Key {
 	public fun encodeTo (Ldev/whyoleg/cryptography/materials/key/KeyFormat;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public static synthetic fun encodeTo$suspendImpl (Ldev/whyoleg/cryptography/materials/key/EncodableKey;Ldev/whyoleg/cryptography/materials/key/KeyFormat;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
@@ -711,12 +727,20 @@ public abstract interface class dev/whyoleg/cryptography/operations/Encryptor {
 }
 
 public abstract interface class dev/whyoleg/cryptography/operations/Hasher {
-	public fun hash (Lkotlinx/io/bytestring/ByteString;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
-	public fun hash ([BLkotlin/coroutines/Continuation;)Ljava/lang/Object;
-	public static synthetic fun hash$suspendImpl (Ldev/whyoleg/cryptography/operations/Hasher;Lkotlinx/io/bytestring/ByteString;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
-	public static synthetic fun hash$suspendImpl (Ldev/whyoleg/cryptography/operations/Hasher;[BLkotlin/coroutines/Continuation;)Ljava/lang/Object;
-	public fun hashBlocking (Lkotlinx/io/bytestring/ByteString;)Lkotlinx/io/bytestring/ByteString;
-	public abstract fun hashBlocking ([B)[B
+	public abstract fun createHashFunction ()Ldev/whyoleg/cryptography/functions/HashFunction;
+	public fun hash (Lkotlinx/io/RawSource;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public fun hash (Lkotlinx/io/bytestring/ByteString;IILkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public fun hash ([BIILkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public static synthetic fun hash$default (Ldev/whyoleg/cryptography/operations/Hasher;Lkotlinx/io/bytestring/ByteString;IILkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
+	public static synthetic fun hash$default (Ldev/whyoleg/cryptography/operations/Hasher;[BIILkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
+	public static synthetic fun hash$suspendImpl (Ldev/whyoleg/cryptography/operations/Hasher;Lkotlinx/io/RawSource;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public static synthetic fun hash$suspendImpl (Ldev/whyoleg/cryptography/operations/Hasher;Lkotlinx/io/bytestring/ByteString;IILkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public static synthetic fun hash$suspendImpl (Ldev/whyoleg/cryptography/operations/Hasher;[BIILkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public fun hashBlocking (Lkotlinx/io/RawSource;)Lkotlinx/io/bytestring/ByteString;
+	public fun hashBlocking (Lkotlinx/io/bytestring/ByteString;II)Lkotlinx/io/bytestring/ByteString;
+	public fun hashBlocking ([BII)[B
+	public static synthetic fun hashBlocking$default (Ldev/whyoleg/cryptography/operations/Hasher;Lkotlinx/io/bytestring/ByteString;IIILjava/lang/Object;)Lkotlinx/io/bytestring/ByteString;
+	public static synthetic fun hashBlocking$default (Ldev/whyoleg/cryptography/operations/Hasher;[BIIILjava/lang/Object;)[B
 }
 
 public abstract interface class dev/whyoleg/cryptography/operations/SecretDerivation {

diff --git a/cryptography-core/api/cryptography-core.klib.api b/cryptography-core/api/cryptography-core.klib.api
@@ -514,6 +514,19 @@ abstract interface dev.whyoleg.cryptography.algorithms/PBKDF2 : dev.whyoleg.cryp
     final object Companion : dev.whyoleg.cryptography/CryptographyAlgorithmId<dev.whyoleg.cryptography.algorithms/PBKDF2> // dev.whyoleg.cryptography.algorithms/PBKDF2.Companion|null[0]
 }
 
+abstract interface dev.whyoleg.cryptography.functions/HashFunction : dev.whyoleg.cryptography.functions/UpdateFunction { // dev.whyoleg.cryptography.functions/HashFunction|null[0]
+    abstract fun hash(): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.functions/HashFunction.hash|hash(){}[0]
+    abstract fun hashInto(kotlin/ByteArray, kotlin/Int = ...): kotlin/Int // dev.whyoleg.cryptography.functions/HashFunction.hashInto|hashInto(kotlin.ByteArray;kotlin.Int){}[0]
+    abstract fun reset() // dev.whyoleg.cryptography.functions/HashFunction.reset|reset(){}[0]
+}
+
+abstract interface dev.whyoleg.cryptography.functions/UpdateFunction : kotlin/AutoCloseable { // dev.whyoleg.cryptography.functions/UpdateFunction|null[0]
+    abstract fun update(kotlin/ByteArray, kotlin/Int = ..., kotlin/Int = ...) // dev.whyoleg.cryptography.functions/UpdateFunction.update|update(kotlin.ByteArray;kotlin.Int;kotlin.Int){}[0]
+    open fun update(kotlinx.io.bytestring/ByteString, kotlin/Int = ..., kotlin/Int = ...) // dev.whyoleg.cryptography.functions/UpdateFunction.update|update(kotlinx.io.bytestring.ByteString;kotlin.Int;kotlin.Int){}[0]
+    open fun updatingSink(kotlinx.io/RawSink): kotlinx.io/RawSink // dev.whyoleg.cryptography.functions/UpdateFunction.updatingSink|updatingSink(kotlinx.io.RawSink){}[0]
+    open fun updatingSource(kotlinx.io/RawSource): kotlinx.io/RawSource // dev.whyoleg.cryptography.functions/UpdateFunction.updatingSource|updatingSource(kotlinx.io.RawSource){}[0]
+}
+
 abstract interface dev.whyoleg.cryptography.materials.key/Key // dev.whyoleg.cryptography.materials.key/Key|null[0]
 
 abstract interface dev.whyoleg.cryptography.materials.key/KeyFormat { // dev.whyoleg.cryptography.materials.key/KeyFormat|null[0]
@@ -564,10 +577,13 @@ abstract interface dev.whyoleg.cryptography.operations/Encryptor { // dev.whyole
 }
 
 abstract interface dev.whyoleg.cryptography.operations/Hasher { // dev.whyoleg.cryptography.operations/Hasher|null[0]
-    abstract fun hashBlocking(kotlin/ByteArray): kotlin/ByteArray // dev.whyoleg.cryptography.operations/Hasher.hashBlocking|hashBlocking(kotlin.ByteArray){}[0]
-    open fun hashBlocking(kotlinx.io.bytestring/ByteString): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hashBlocking|hashBlocking(kotlinx.io.bytestring.ByteString){}[0]
-    open suspend fun hash(kotlin/ByteArray): kotlin/ByteArray // dev.whyoleg.cryptography.operations/Hasher.hash|hash(kotlin.ByteArray){}[0]
-    open suspend fun hash(kotlinx.io.bytestring/ByteString): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hash|hash(kotlinx.io.bytestring.ByteString){}[0]
+    abstract fun createHashFunction(): dev.whyoleg.cryptography.functions/HashFunction // dev.whyoleg.cryptography.operations/Hasher.createHashFunction|createHashFunction(){}[0]
+    open fun hashBlocking(kotlin/ByteArray, kotlin/Int = ..., kotlin/Int = ...): kotlin/ByteArray // dev.whyoleg.cryptography.operations/Hasher.hashBlocking|hashBlocking(kotlin.ByteArray;kotlin.Int;kotlin.Int){}[0]
+    open fun hashBlocking(kotlinx.io.bytestring/ByteString, kotlin/Int = ..., kotlin/Int = ...): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hashBlocking|hashBlocking(kotlinx.io.bytestring.ByteString;kotlin.Int;kotlin.Int){}[0]
+    open fun hashBlocking(kotlinx.io/RawSource): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hashBlocking|hashBlocking(kotlinx.io.RawSource){}[0]
+    open suspend fun hash(kotlin/ByteArray, kotlin/Int = ..., kotlin/Int = ...): kotlin/ByteArray // dev.whyoleg.cryptography.operations/Hasher.hash|hash(kotlin.ByteArray;kotlin.Int;kotlin.Int){}[0]
+    open suspend fun hash(kotlinx.io.bytestring/ByteString, kotlin/Int = ..., kotlin/Int = ...): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hash|hash(kotlinx.io.bytestring.ByteString;kotlin.Int;kotlin.Int){}[0]
+    open suspend fun hash(kotlinx.io/RawSource): kotlinx.io.bytestring/ByteString // dev.whyoleg.cryptography.operations/Hasher.hash|hash(kotlinx.io.RawSource){}[0]
 }
 
 abstract interface dev.whyoleg.cryptography.operations/SecretDerivation { // dev.whyoleg.cryptography.operations/SecretDerivation|null[0]

diff --git a/cryptography-core/src/commonMain/kotlin/functions/HashFunction.kt b/cryptography-core/src/commonMain/kotlin/functions/HashFunction.kt
@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2024 Oleg Yukhnevich. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package dev.whyoleg.cryptography.functions
+
+import kotlinx.io.bytestring.*
+
+public interface HashFunction : UpdateFunction {
+    public fun hash(): ByteString
+    public fun hashInto(destination: ByteArray, destinationOffset: Int = 0): Int
+    public fun reset()
+}
diff --git a/cryptography-core/src/commonMain/kotlin/functions/UpdateFunction.kt b/cryptography-core/src/commonMain/kotlin/functions/UpdateFunction.kt
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2024 Oleg Yukhnevich. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package dev.whyoleg.cryptography.functions
+
+import dev.whyoleg.cryptography.*
+import kotlinx.io.*
+import kotlinx.io.bytestring.*
+import kotlinx.io.unsafe.*
+
+public interface UpdateFunction : AutoCloseable {
+    public fun update(source: ByteArray, startIndex: Int = 0, endIndex: Int = source.size)
+    public fun update(source: ByteString, startIndex: Int = 0, endIndex: Int = source.size) {
+        update(source.asByteArray(), startIndex, endIndex)
+    }
+
+    public fun updatingSource(source: RawSource): RawSource = UpdatingSource(this, source)
+    public fun updatingSink(sink: RawSink): RawSink = UpdatingSink(this, sink)
+}
+
+private class UpdatingSource(
+    private val function: UpdateFunction,
+    private val source: RawSource,
+) : RawSource {
+    override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
+        val result = source.readAtMostTo(sink, byteCount)
+        if (result != -1L) {
+            @OptIn(UnsafeIoApi::class)
+            UnsafeBufferOperations.iterate(sink, sink.size - result) { context, head, _ ->
+                var segment = head
+                while (segment != null) {
+                    context.withData(segment, function::update)
+                    segment = context.next(segment)
+                }
+            }
+        }
+        return result
+    }
+
+    override fun close(): Unit = source.close()
+}
+
+private class UpdatingSink(
+    private val function: UpdateFunction,
+    private val sink: RawSink,
+) : RawSink {
+    override fun write(source: Buffer, byteCount: Long) {
+        source.require(byteCount)
+
+        @OptIn(UnsafeIoApi::class)
+        UnsafeBufferOperations.iterate(source) { context, head ->
+            var consumedCount = 0L
+            var segment = head
+            while (segment != null && consumedCount < byteCount) {
+                context.withData(segment) { bytes, startIndex, endIndex ->
+                    val toUpdate = minOf(byteCount - consumedCount, (endIndex - startIndex).toLong()).toInt()
+                    function.update(bytes, startIndex, startIndex + toUpdate)
+                    consumedCount += toUpdate
+                }
+                segment = context.next(segment)
+            }
+        }
+
+        sink.write(source, byteCount)
+    }
+
+    override fun flush(): Unit = sink.flush()
+    override fun close(): Unit = sink.close()
+}
diff --git a/cryptography-core/src/commonMain/kotlin/operations/Hasher.kt b/cryptography-core/src/commonMain/kotlin/operations/Hasher.kt
@@ -5,13 +5,37 @@
 package dev.whyoleg.cryptography.operations
 
 import dev.whyoleg.cryptography.*
+import dev.whyoleg.cryptography.functions.*
+import kotlinx.io.*
 import kotlinx.io.bytestring.*
 
 @SubclassOptInRequired(CryptographyProviderApi::class)
 public interface Hasher {
-    public suspend fun hash(data: ByteArray): ByteArray = hashBlocking(data)
-    public fun hashBlocking(data: ByteArray): ByteArray
+    public fun createHashFunction(): HashFunction
 
-    public suspend fun hash(data: ByteString): ByteString = hash(data.asByteArray()).asByteString()
-    public fun hashBlocking(data: ByteString): ByteString = hashBlocking(data.asByteArray()).asByteString()
+    public suspend fun hash(data: ByteArray, startIndex: Int = 0, endIndex: Int = data.size): ByteArray {
+        return hashBlocking(data, startIndex, endIndex)
+    }
+
+    public suspend fun hash(data: ByteString, startIndex: Int = 0, endIndex: Int = data.size): ByteString {
+        return hash(data.asByteArray(), startIndex, endIndex).asByteString()
+    }
+
+    public suspend fun hash(data: RawSource): ByteString {
+        return hashBlocking(data)
+    }
+
+    public fun hashBlocking(data: ByteArray, startIndex: Int = 0, endIndex: Int = data.size): ByteArray = createHashFunction().use {
+        it.update(data, startIndex, endIndex)
+        it.hash().asByteArray()
+    }
+
+    public fun hashBlocking(data: ByteString, startIndex: Int = 0, endIndex: Int = data.size): ByteString {
+        return hashBlocking(data.asByteArray(), startIndex, endIndex).asByteString()
+    }
+
+    public fun hashBlocking(data: RawSource): ByteString = createHashFunction().use {
+        it.updatingSource(data).buffered().transferTo(discardingSink())
+        it.hash()
+    }
 }
diff --git a/cryptography-providers-tests-api/src/commonMain/kotlin/support.kt b/cryptography-providers-tests-api/src/commonMain/kotlin/support.kt
@@ -12,6 +12,13 @@ import dev.whyoleg.cryptography.serialization.asn1.modules.*
 import dev.whyoleg.cryptography.serialization.pem.*
 import kotlinx.io.bytestring.*
 
+fun AlgorithmTestScope<*>.supportsFunctions() = supports {
+    when {
+        provider.isWebCrypto -> "Incremental functions"
+        else                 -> null
+    }
+}
+
 fun AlgorithmTestScope<*>.supportsDigest(digest: CryptographyAlgorithmId<Digest>): Boolean = supports {
     val sha3Algorithms = setOf(SHA3_224, SHA3_256, SHA3_384, SHA3_512)
     when {

diff --git a/cryptography-providers-tests/src/commonMain/kotlin/default/DigestTest.kt b/cryptography-providers-tests/src/commonMain/kotlin/default/DigestTest.kt
@@ -8,6 +8,8 @@ import dev.whyoleg.cryptography.*
 import dev.whyoleg.cryptography.algorithms.*
 import dev.whyoleg.cryptography.providers.tests.api.*
 import dev.whyoleg.cryptography.random.*
+import kotlinx.io.*
+import kotlinx.io.bytestring.*
 import kotlin.math.*
 import kotlin.test.*
 
@@ -57,4 +59,61 @@ abstract class DigestTest(provider: CryptographyProvider) : ProviderTest(provide
 
     @Test
     fun testSHA3_512() = test(SHA3_512, 64)
+
+    @Test
+    fun testIndexes() = testAlgorithm(SHA256) {
+        val hasher = algorithm.hasher()
+
+        val array = ByteArray(10)
+
+        assertFails { hasher.hash(array, -1, 10) }
+        assertFails { hasher.hash(array, 0, -1) }
+        assertFails { hasher.hash(array, 20, 10) }
+        assertFails { hasher.hash(array, 0, 20) }
+
+        if (!supportsFunctions()) return@testAlgorithm
+
+        val hashFunction = algorithm.hasher().createHashFunction()
+
+        assertFails { hashFunction.update(array, -1, 10) }
+        assertFails { hashFunction.update(array, 0, -1) }
+        assertFails { hashFunction.update(array, 20, 10) }
+        assertFails { hashFunction.update(array, 0, 20) }
+
+        hashFunction.update(array)
+    }
+
+    @Test
+    fun testFunctionChunked() = testAlgorithm(SHA256) {
+        if (!supportsFunctions()) return@testAlgorithm
+
+        val hasher = algorithm.hasher()
+        val bytes = ByteString(CryptographyRandom.nextBytes(10000))
+
+        val digest = hasher.hash(bytes)
+        hasher.createHashFunction().use { function ->
+            repeat(10) {
+                function.update(bytes, it * 1000, (it + 1) * 1000)
+            }
+            assertContentEquals(digest, function.hash())
+        }
+    }
+
+    @Test
+    fun testFunctionSource() = testAlgorithm(SHA256) {
+        val hasher = algorithm.hasher()
+
+        val bytes = ByteString(CryptographyRandom.nextBytes(10000))
+        val source = Buffer()
+        source.write(bytes)
+        val digest = hasher.hash(bytes)
+
+        assertContentEquals(digest, hasher.hash(source.copy()))
+
+        if (!supportsFunctions()) return@testAlgorithm
+        hasher.createHashFunction().use { function ->
+            assertContentEquals(bytes, function.updatingSource(source).buffered().readByteString())
+            assertContentEquals(digest, function.hash())
+        }
+    }
 }
diff --git a/cryptography-providers/apple/src/commonMain/kotlin/algorithms/CCDigest.kt b/cryptography-providers/apple/src/commonMain/kotlin/algorithms/CCDigest.kt
@@ -6,24 +6,62 @@ package dev.whyoleg.cryptography.providers.apple.algorithms
 
 import dev.whyoleg.cryptography.*
 import dev.whyoleg.cryptography.algorithms.*
+import dev.whyoleg.cryptography.functions.*
 import dev.whyoleg.cryptography.operations.*
 import dev.whyoleg.cryptography.providers.apple.internal.*
 import kotlinx.cinterop.*
+import kotlinx.io.bytestring.*
+import kotlinx.io.bytestring.unsafe.*
 
-internal class CCDigest(
-    private val hashAlgorithm: CCHashAlgorithm,
+internal class CCDigest<CTX : CPointed>(
+    private val hashAlgorithm: CCHashAlgorithm<CTX>,
     override val id: CryptographyAlgorithmId<Digest>,
 ) : Hasher, Digest {
     override fun hasher(): Hasher = this
+    override fun createHashFunction(): HashFunction = CCHashFunction(
+        algorithm = hashAlgorithm,
+        context = Resource(hashAlgorithm.alloc(), nativeHeap::free)
+    )
+}
+
+private class CCHashFunction<CTX : CPointed>(
+    private val algorithm: CCHashAlgorithm<CTX>,
+    private val context: Resource<CPointer<CTX>>,
+) : HashFunction, SafeCloseable(SafeCloseAction(context, AutoCloseable::close)) {
+    init {
+        reset()
+    }
+
+    override fun update(source: ByteArray, startIndex: Int, endIndex: Int) {
+        checkBounds(source.size, startIndex, endIndex)
+
+        val context = context.access()
+        source.usePinned {
+            check(algorithm.ccUpdate(context, it.safeAddressOf(startIndex), (endIndex - startIndex).convert()) > 0)
+        }
+    }
+
+    override fun hashInto(destination: ByteArray, destinationOffset: Int): Int {
+        checkBounds(destination.size, destinationOffset, destinationOffset + algorithm.digestSize)
+
+        val context = context.access()
+        destination.usePinned {
+            check(algorithm.ccFinal(context, it.safeAddressOf(destinationOffset).reinterpret()) > 0)
+        }
+        reset()
+        return algorithm.digestSize
+    }
+
+    @OptIn(UnsafeByteStringApi::class)
+    override fun hash(): ByteString {
+        val output = ByteArray(algorithm.digestSize)
+        hashInto(output)
+        return UnsafeByteStringOperations.wrapUnsafe(output)
+    }
 
-    @OptIn(ExperimentalUnsignedTypes::class)
-    override fun hashBlocking(data: ByteArray): ByteArray {
-        val output = ByteArray(hashAlgorithm.digestSize)
-        hashAlgorithm.ccHash(
-            data = data.fixEmpty().refTo(0),
-            dataLength = data.size.convert(),
-            digest = output.asUByteArray().refTo(0)
-        )
-        return output
+    // TODO: recheck how it should work
+    override fun reset() {
+        val context = context.access()
+        check(algorithm.ccInit(context) > 0)
     }
 }