feat(abi)!: add an option to read and write raw bytes to an abi to allow for more complex structs to be encoded/decoded.

.changeset/stupid-years-collect.md

@@ -0,0 +1,5 @@
+---
+'@axelar-network/axelar-cgp-sui': minor
+---
+
+Add read_raw_bytes and write_raw_bytes to abi to support more data formats.

move/abi/README.md

@@ -20,6 +20,7 @@ This type can be used to encode abi data. It has the following relevant function
 - `abi::write_bytes(self: &mut AbiWriter, var: vector<u8>): &mut AbiWriter`: Writes the provided bytes into the next slot in the `AbiWriter`. This should be used to write all types that are equivelant to `vector<u8>` (`ascii::String` and `string::String` for example) by converting them to `vector<u8>`.
 - `abi::write_vector_u256(self: &mut AbiWriter, var: vector<u256>,): &mut AbiWriter`: Writes the provided `vector<u256>` into the next slot in the  `AbiWriter`. This should be used for vectors of other fixed length variables as well.
 - `abi::write_vector_bytes(self: &mut AbiWriter, var: vector<vector<u8>>,): &mut AbiWriter`: Writes the provided `vector<vector<u8>>` into the nexts slot in the `AbiWriter`. This should be used for vectors of other variable length variables as well.
+- `abi::write_bytes_raw(self: &mut AbiWriter, var: vector<u8>,): &mut AbiWriter`: Writes the raw bytes provided to the next slot of the `AbiWriter`. These bytes are not length prefixed, and can therefore not be decoded as bytes. The purpose of this function is to allow for encoding of more complex, unavailable structs.
 
 #### Example
 ```rust
@@ -33,7 +34,33 @@ let encoded_data = writer.into_bytes();
 ```
 
 #### More complex types
-More complex types are not supported yet.
+More complex types are curently not supported. This is because Sui Move does not support any sort of type inspection (like `is_vector<T>`) to recursively encode vectors. However with `abi::write_bytes_raw` these types can be encoded with some extra work from the user.
+For example to encode a struct consisting of  `u256` called `number` and a `vector<u8>` called `data` into an `AbiWriter` named `writer` a user could do
+```rust
+let mut struct_writer = new_writer(2);
+struct_writer
+    .write_u256(number)
+    .write_bytes(data);
+writer
+    .write_bytes_raw(struct_writer.into_bytes());
+```
+As another example, to abi encode a `vector<vector<u256>>` named `table` into an `AbiWriter` named `writer` a user could do
+```rust
+let length = table.length();
+
+let mut length_writer = new_writer(1);
+length_writer.write_u256(length as u256);
+let mut bytes = length_writer.into_bytes();
+
+let mut table_writer = new_writer(length);
+table.do!(|row| {
+    table_writer.write_vector_u256(row);
+});
+bytes.append(table_writer.into_bytes());
+
+writer
+    .write_bytes_raw(bytes);
+```
 
 ### `AbiReader`
 
@@ -46,6 +73,7 @@ This type can be used to decode abi enocded data. The relevant functions are as
 - `abi::read_bytes(self: &mut AbiReader): vector<8>`: Read a `vector<u8>` from the next slot of the `AbiReader`. Should be used to read other variable length types as well.
 - `abi::read_vector_u256(self: &mut AbiReader): vector<u256>`: Read a `vector<u256>` from the next slot of the `AbiReader`. Should be used to read other fixed length types as well.
 - `abi::read_vector_bytes(self: &mut AbiReader): vector<vector<u8>>`: Read a `vector<vector<u8>>` from the next slot of the `AbiReader`. Should be used to read other vectors of variable length types as well (such as `vector<ascii::String>`).
+- `abi::read_bytes_raw(self: &mut AbiReader): vector<u8>`: Read the raw bytes encoded in the next slot of the `AbiReader`. This will include any bytes encoded after the raw bytes desired which should be ignored.
 
 #### Example
 ```rust
@@ -58,4 +86,27 @@ let info = reader.read_vector_bytes();
 
 #### More Complex Types
 
-More complex types are not supported yet.
+For more complex types like structs or nested vectors `read_bytes_raw` can be used and decoded. For to read a struct that contains a `u256` and a `vector<u8>` from an `AbiReader` called `reader` a user may:
+```rust
+    let struct_bytes = reader.read_bytes_raw();
+
+    let mut struct_reader = new_reader(struct_bytes);
+    let number = struct_reader.read_u256();
+    let data = struct_reader.read_bytes();
+```
+As another example, to decode a `vector<vector<u256>>` into a variable called table from an `AbiReader` called `reader` a user can:
+```rust
+let mut table_bytes = reader.read_bytes_raw();
+
+// Split the data into the lenth and the actual table contents
+let length_bytes = vector::tabulate!(U256_BYTES, |_| table_bytes.remove(0));
+
+let mut length_reader = new_reader(length_bytes);
+let length = length_reader.read_u256() as u64;
+
+let mut table_reader = new_reader(table_bytes);
+let table = vector::tabulate!(
+    length,
+    |_| table_reader.read_vector_u256(),
+);
+```

move/abi/sources/abi.move

@@ -154,6 +154,40 @@ public fun read_vector_bytes(self: &mut AbiReader): vector<vector<u8>> {
     var
 }
 
+/// Reads the raw bytes of a variable length variable. This will return
+/// additional bytes at the end of the structure as there is no way to know how
+/// to decode the bytes returned. This can be used to decode structs and complex
+/// nested vectors that this library does not provide a method for.
+/// For more complex types like structs or nested vectors `read_bytes_raw` can
+/// be used and decoded manualy. To read a struct that contains a `u256` and
+/// a `vector<u8>` from an `AbiReader` called `reader` a user may:
+/// ```rust
+/// let struct_bytes = reader.read_bytes_raw();
+/// let mut struct_reader = new_reader(struct_bytes);
+/// let number = struct_reader.read_u256();
+/// let data = struct_reader.read_bytes();
+/// ```
+/// As another example, to decode a `vector<vector<u256>>` into a variable
+/// called `table` from an `AbiReader` called `reader` a user can:
+/// ```rust
+/// let mut table_bytes = reader.read_bytes_raw();
+/// // Split the data into the lenth and the actual table contents
+/// let length_bytes = vector::tabulate!(U256_BYTES, |_| table_bytes.remove(0));
+/// let mut length_reader = new_reader(length_bytes);
+/// let length = length_reader.read_u256() as u64;
+/// let mut table_reader = new_reader(table_bytes);
+/// let table = vector::tabulate!(length, |_| table_reader.read_vector_u256());
+/// ```
+public fun read_bytes_raw(self: &mut AbiReader): vector<u8> {
+    // Move position to the start of the bytes
+    let offset = self.read_u256() as u64;
+    let length = self.bytes.length() - offset;
+
+    let var = vector::tabulate!(length, |i| self.bytes[offset + i]);
+
+    var
+}
+
 /// Write a `u256` into the next slot of an `AbiWriter`. Can be used to write
 /// other fixed lenght variables as well.
 public fun write_u256(self: &mut AbiWriter, var: u256): &mut AbiWriter {
@@ -229,6 +263,46 @@ public fun write_vector_bytes(
     self
 }
 
+/// Write raw bytes to the next slot of an `AbiWriter`. This can be used to
+/// write structs or more nested arrays that we support in this module.
+/// For example to encode a struct consisting of  `u256` called `number` and a
+/// `vector<u8>` called `data` into an `AbiWriter` named `writer` a user could
+/// do
+/// ```rust
+/// let mut struct_writer = new_writer(2);
+/// struct_writer
+///     .write_u256(number)
+///     .write_bytes(data);
+/// writer
+///     .write_bytes_raw(struct_writer.into_bytes());
+/// ```
+/// As another example, to abi encode a `vector<vector<u256>>` named `table`
+/// into an `AbiWriter` named `writer` a user could do
+/// ```rust
+/// let length = table.length();
+/// let mut length_writer = new_writer(1);
+/// length_writer.write_u256(length as u256);
+/// let mut bytes = length_writer.into_bytes();
+/// let mut table_writer = new_writer(length);
+/// table.do!(|row| {
+///     table_writer.write_vector_u256(row);
+/// });
+/// bytes.append(table_writer.into_bytes());
+/// writer
+///     .write_bytes_raw(bytes);
+/// ```
+public fun write_bytes_raw(
+    self: &mut AbiWriter,
+    var: vector<u8>,
+): &mut AbiWriter {
+    let offset = self.bytes.length() as u256;
+    self.write_u256(offset);
+
+    self.append_bytes(var);
+
+    self
+}
+
 // ------------------
 // Internal Functions
 // ------------------
@@ -246,7 +320,8 @@ fun append_bytes(self: &mut AbiWriter, var: vector<u8>) {
 
     self.bytes.append(var);
 
-    ((U256_BYTES) - 1 - (length - 1) % U256_BYTES).do!(|_| self.bytes.push_back(0));
+    let pad_length = (U256_BYTES) - 1 - (length - 1) % U256_BYTES;
+    pad_length.do!(|_| self.bytes.push_back(0));
 }
 
 fun decode_bytes(self: &mut AbiReader): vector<u8> {
@@ -366,3 +441,65 @@ fun test_append_empty_bytes() {
     let mut writer = new_writer(0);
     writer.append_bytes(vector[]);
 }
+
+#[test]
+fun test_raw_struct() {
+    let number = 3;
+    let data = b"data";
+
+    let mut writer = new_writer(1);
+    let mut struct_writer = new_writer(2);
+    struct_writer.write_u256(number).write_bytes(data);
+    writer.write_bytes_raw(struct_writer.into_bytes());
+
+    let bytes = writer.into_bytes();
+
+    let mut reader = new_reader(bytes);
+
+    let struct_bytes = reader.read_bytes_raw();
+
+    let mut struct_reader = new_reader(struct_bytes);
+
+    assert!(struct_reader.read_u256() == number);
+    assert!(struct_reader.read_bytes() == data);
+}
+
+#[test]
+fun test_raw_table() {
+    let table = vector[vector[1, 2, 3], vector[4, 5, 6]];
+
+    let mut writer = new_writer(1);
+
+    let length = table.length();
+
+    let mut length_writer = new_writer(1);
+    length_writer.write_u256(length as u256);
+    let mut bytes = length_writer.into_bytes();
+
+    let mut table_writer = new_writer(length);
+    table.do!(|row| {
+        table_writer.write_vector_u256(row);
+    });
+    bytes.append(table_writer.into_bytes());
+
+    writer.write_bytes_raw(bytes);
+
+    let bytes = writer.into_bytes();
+
+    let mut reader = new_reader(bytes);
+    let mut table_bytes = reader.read_bytes_raw();
+
+    // Split the data into the lenth and the actual table contents
+    let length_bytes = vector::tabulate!(U256_BYTES, |_| table_bytes.remove(0));
+
+    let mut length_reader = new_reader(length_bytes);
+    let length = length_reader.read_u256() as u64;
+
+    let mut table_reader = new_reader(table_bytes);
+    let table_read = vector::tabulate!(
+        length,
+        |_| table_reader.read_vector_u256(),
+    );
+
+    assert!(table_read == table);
+}

test/testdata/interface_abi_abi.json

@@ -64,14 +64,6 @@
       },
       "returnType": "vector<u8>"
     },
-    "into_remaining_bytes": {
-      "name": "into_remaining_bytes",
-      "visibility": "public",
-      "params": {
-        "self#0#0": "AbiReader"
-      },
-      "returnType": "vector<u8>"
-    },
     "read_u256": {
       "name": "read_u256",
       "visibility": "public",
@@ -112,6 +104,14 @@
       },
       "returnType": "vector<vector<u8>>"
     },
+    "read_bytes_raw": {
+      "name": "read_bytes_raw",
+      "visibility": "public",
+      "params": {
+        "self#0#0": "&mut AbiReader"
+      },
+      "returnType": "vector<u8>"
+    },
     "write_u256": {
       "name": "write_u256",
       "visibility": "public",
@@ -156,6 +156,15 @@
         "var#0#0": "vector<vector<u8>>"
       },
       "returnType": "&mut AbiWriter"
+    },
+    "write_bytes_raw": {
+      "name": "write_bytes_raw",
+      "visibility": "public",
+      "params": {
+        "self#0#0": "&mut AbiWriter",
+        "var#0#0": "vector<u8>"
+      },
+      "returnType": "&mut AbiWriter"
     }
   }
 }