Merge branch 'main' into user/ashdhin/IssueTemplateRegressionCheckbox

.builder/actions/crt_size_check.py

@@ -11,7 +11,7 @@ def run(self, env):
         # Maximum package size (for current platform) in bytes
         # NOTE: if you increase this, you might also need to increase the
         # limit in continuous-delivery/pack.sh
-        max_size = 7_500_000
+        max_size = 8_000_000
         # size of current folder
         folder_size = 0
         # total size in bytes

.github/workflows/ci.yml

@@ -167,7 +167,7 @@ jobs:
   check-docs:
     runs-on: ubuntu-20.04 # latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Check docs
@@ -179,7 +179,7 @@ jobs:
     runs-on: ubuntu-20.04 # latest
     steps:
     - name: Checkout Source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
@@ -191,7 +191,7 @@ jobs:
   check-lockfile-version:
     runs-on: ubuntu-20.04 # latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Check for edits to package-lock.json

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.9)
 project(aws-crt-nodejs C)
 option(BUILD_DEPS "Builds aws common runtime dependencies as part of build, only do this if you don't want to control your dependency chain." ON)
 
@@ -45,7 +45,7 @@ if (BUILD_DEPS)
     if (UNIX AND NOT APPLE)
         include(AwsPrebuildDependency)
         # s2n-tls uses libcrypto during its configuration, so we need to prebuild aws-lc.
-        prebuild_dependency(
+        aws_prebuild_dependency(
             DEPENDENCY_NAME AWSLC
             SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/crt/aws-lc
             CMAKE_ARGUMENTS
@@ -54,8 +54,14 @@ if (BUILD_DEPS)
                 -DBUILD_LIBSSL=OFF
                 -DBUILD_TESTING=OFF
         )
-        set(UNSAFE_TREAT_WARNINGS_AS_ERRORS OFF)
-        add_subdirectory(crt/s2n)
+        # prebuild s2n-tls.
+        aws_prebuild_dependency(
+            DEPENDENCY_NAME S2N
+            SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/crt/s2n
+            CMAKE_ARGUMENTS
+                -DUNSAFE_TREAT_WARNINGS_AS_ERRORS=OFF
+                -DBUILD_TESTING=OFF
+        )
     endif()
 
     add_subdirectory(crt/aws-c-sdkutils)
@@ -74,10 +80,6 @@ else()
     set(IN_SOURCE_BUILD OFF)
 endif()
 
-if (POLICY CMP0069)
-    cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
-endif()
-
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_PREFIX_PATH}/${CMAKE_INSTALL_LIBDIR}/cmake")
 
 include(AwsCFlags)

continuous-delivery/pack.sh

@@ -39,7 +39,7 @@ mkdir $UNZIP
 tar -xf aws-crt-$CURRENT_TAG.tgz -C $UNZIP
 PACK_FILE_SIZE_KB=$(du -sk $UNZIP | awk '{print $1}')
 echo "Current package size: ${PACK_FILE_SIZE_KB}"
-if expr $PACK_FILE_SIZE_KB \> "$((25000))" ; then
+if expr $PACK_FILE_SIZE_KB \> "$((26000))" ; then
     # the package size is too large, return -1
     echo "Package size is too large!"
     exit -1

lib/browser/crypto.ts

@@ -12,25 +12,35 @@
  */
 
 import * as Crypto from "crypto-js";
-import { fromUtf8 } from "@aws-sdk/util-utf8-browser";
 import { Hashable } from "../common/crypto";
 
 export { Hashable } from "../common/crypto";
 
 /**
- * Object that allows for continuous MD5 hashing of data.
- *
- * @category Crypto
+ * CryptoJS does not provide easy access to underlying bytes.
+ * As a workaround just dump it to a string and then reinterpret chars as individual bytes.
+ * Note: we are using Latin1 here because its a static sized 8 bit encoding so each char maps directly to a byte value.
+ * TODO: long term we would probably want to move to WebCrypto for SHA's and some other 3p for crc's and md5.
+ * @param hash 
+ * @returns 
  */
-export class Md5Hash {
-    private hash?: Crypto.WordArray;
+function wordArrayToUint8Array(hash: Crypto.WordArray) {
+    return Uint8Array.from(hash.toString(Crypto.enc.Latin1).split('').map(c => c.charCodeAt(0)));;
+}
+
+class BaseHash {
+    private hasher : any;
+
+    constructor(hasher: any) {
+        this.hasher = hasher;
+    }
 
     /**
      * Hashes additional data
      * @param data Additional data to hash
      */
     update(data: Hashable) {
-        this.hash = Crypto.MD5(data.toString(), this.hash ? this.hash.toString() : undefined);
+        this.hasher.update(data.toString());
     }
 
     /**
@@ -41,10 +51,20 @@ export class Md5Hash {
      * @returns the final hash digest
      */
     finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+        const hashBuffer = wordArrayToUint8Array(this.hasher.finalize()) ;
+        const truncated = hashBuffer.slice(0, truncate_to ? truncate_to : hashBuffer.length);
+        return new DataView(truncated.buffer);;
+    }
+}
+
+/**
+ * Object that allows for continuous MD5 hashing of data.
+ *
+ * @category Crypto
+ */
+export class Md5Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.MD5.create());
     }
 }
 
@@ -71,29 +91,9 @@ export function hash_md5(data: Hashable, truncate_to?: number): DataView {
  *
  * @category Crypto
  */
-export class Sha256Hash {
-    private hash?: Crypto.WordArray;
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hash = Crypto.SHA256(data.toString(), this.hash ? this.hash.toString() : undefined);
-    }
-
-    /**
-     * Completes the hash computation and returns the final hash digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hash digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+export class Sha256Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.SHA256.create());
     }
 }
 
@@ -109,40 +109,19 @@ export class Sha256Hash {
  * @category Crypto
  */
 export function hash_sha256(data: Hashable, truncate_to?: number): DataView {
-    const digest = Crypto.SHA256(data.toString()).toString();
-    const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-    const bytes = fromUtf8(truncated);
-    return new DataView(bytes.buffer);
+    const sha256 = new Sha256Hash();
+    sha256.update(data);
+    return sha256.finalize(truncate_to);
 }
 
 /**
  * Object that allows for continuous SHA1 hashing of data.
  *
  * @category Crypto
  */
- export class Sha1Hash {
-    private hash?: Crypto.WordArray;
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hash = Crypto.SHA1(data.toString(), this.hash ? this.hash.toString() : undefined);
-    }
-
-    /**
-     * Completes the hash computation and returns the final hash digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hash digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hash ? this.hash.toString() : '';
-        const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+ export class Sha1Hash extends BaseHash {
+    constructor() {
+        super(Crypto.algo.SHA1.create());
     }
 }
 
@@ -158,49 +137,24 @@ export function hash_sha256(data: Hashable, truncate_to?: number): DataView {
  * @category Crypto
  */
 export function hash_sha1(data: Hashable, truncate_to?: number): DataView {
-    const digest = Crypto.SHA1(data.toString()).toString();
-    const truncated = digest.substring(0, truncate_to ? truncate_to : digest.length);
-    const bytes = fromUtf8(truncated);
-    return new DataView(bytes.buffer);
+    const sha1 = new Sha1Hash();
+    sha1.update(data);
+    return sha1.finalize(truncate_to);
 }
 
 /**
  * Object that allows for continuous hashing of data with an hmac secret.
  *
  * @category Crypto
  */
-export class Sha256Hmac {
-    private hmac: any;
-
+export class Sha256Hmac extends BaseHash {
     /**
      * Constructor for the Sha256Hmac class type
      * @param secret secret key to seed the hmac process with
      */
     constructor(secret: Hashable) {
         // @ts-ignore types file doesn't have this signature of create()
-        this.hmac = Crypto.algo.HMAC.create(Crypto.algo.SHA256, secret);
-    }
-
-    /**
-     * Hashes additional data
-     * @param data Additional data to hash
-     */
-    update(data: Hashable) {
-        this.hmac.update(data.toString());
-    }
-
-    /**
-     * Completes the hash computation and returns the final hmac digest.
-     *
-     * @param truncate_to The maximum number of bytes to receive. Leave as undefined or 0 to receive the entire digest.
-     *
-     * @returns the final hmac digest
-     */
-    finalize(truncate_to?: number): DataView {
-        const digest = this.hmac.finalize();
-        const truncated = digest.toString().substring(0, truncate_to ? truncate_to : digest.length);
-        const bytes = fromUtf8(truncated);
-        return new DataView(bytes.buffer);
+        super(Crypto.algo.HMAC.create(Crypto.algo.SHA256, secret));
     }
 }
 

lib/native/binding.d.ts

@@ -142,6 +142,8 @@ export function hmac_sha256_compute(secret: StringLike, data: StringLike, trunca
 export function checksums_crc32(data: StringLike, previous?: number): number;
 /** @internal */
 export function checksums_crc32c(data: StringLike, previous?: number): number;
+/** @internal */
+export function checksums_crc64nvme(data: StringLike, previous?: DataView): DataView;
 
 /* MQTT5 Client */
 

lib/native/checksums.spec.ts

@@ -81,4 +81,21 @@ test('crc32c_large_buffer', () => {
     const output = checksums.crc32c(arr);
     const expected = 0xfb5b991d
     expect(output).toEqual(expected);
-});
+});
+
+test('crc64nvme_zeros_one_shot', () => {
+    const arr = new Uint8Array(32);
+    const output = checksums.crc64nvme(arr);
+    expect(output.getBigUint64(0)).toEqual(BigInt("0xCF3473434D4ECF3B"));
+});
+
+test('crc64nvme_zeros_iterated', () => {
+    const buffer = new ArrayBuffer(8);
+    let previous = new DataView(buffer);
+    previous.setBigUint64(0, BigInt(0));
+
+    for (let i = 0; i < 32; i++) {
+        previous = checksums.crc64nvme(new Uint8Array(1), previous);
+    }
+    expect(previous.getBigUint64(0)).toEqual(BigInt("0xCF3473434D4ECF3B"));
+});

lib/native/checksums.ts

@@ -38,4 +38,16 @@ export function crc32(data: Hashable, previous?: number): number {
  */
  export function crc32c(data: Hashable, previous?: number): number {
     return crt_native.checksums_crc32c(data, previous);
-}
+}
+
+/**
+ * Computes a crc64nvme checksum.
+ *
+ * @param data The data to checksum
+ * @param previous previous crc64nvme checksum result. Used if you are buffering large input.
+ *
+ * @category Crypto
+ */
+export function crc64nvme(data: Hashable, previous?: DataView): DataView {
+    return crt_native.checksums_crc64nvme(data, previous);
+}