Blockstream · shesek · Feb 16, 2024 · Feb 15, 2024
diff --git a/src/bin/popular-scripts.rs b/src/bin/popular-scripts.rs
@@ -1,9 +1,9 @@
 extern crate electrs;
 
-use bincode::Options;
 use electrs::{
     config::Config,
     new_index::{Store, TxHistoryKey},
+    util::bincode,
 };
 use hex::DisplayHex;
 
@@ -24,10 +24,8 @@ fn main() {
             break;
         }
 
-        let entry: TxHistoryKey = bincode::options()
-            .with_big_endian()
-            .deserialize(&key)
-            .expect("failed to deserialize TxHistoryKey");
+        let entry: TxHistoryKey =
+            bincode::deserialize_big(&key).expect("failed to deserialize TxHistoryKey");
 
         if curr_scripthash != entry.hash {
             if total_entries > 100 {

diff --git a/src/elements/asset.rs b/src/elements/asset.rs
@@ -13,7 +13,7 @@ use crate::elements::registry::{AssetMeta, AssetRegistry};
 use crate::errors::*;
 use crate::new_index::schema::{TxHistoryInfo, TxHistoryKey, TxHistoryRow};
 use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query};
-use crate::util::{full_hash, Bytes, FullHash, TransactionStatus, TxInput};
+use crate::util::{bincode, full_hash, Bytes, FullHash, TransactionStatus, TxInput};
 
 lazy_static! {
     pub static ref NATIVE_ASSET_ID: AssetId =
@@ -192,7 +192,7 @@ pub fn index_confirmed_tx_assets(
     // reissuances are only kept under the history index.
     rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow {
         key: [b"i", &asset_id.into_inner()[..]].concat(),
-        value: bincode::serialize(&asset_row).unwrap(),
+        value: bincode::serialize_little(&asset_row).unwrap(),
     }));
 }
 
@@ -371,7 +371,7 @@ pub fn lookup_asset(
 
     let chain_row = history_db
         .get(&[b"i", &asset_id.into_inner()[..]].concat())
-        .map(|row| bincode::deserialize::<AssetRow>(&row).expect("failed parsing AssetRow"));
+        .map(|row| bincode::deserialize_little::<AssetRow>(&row).expect("failed parsing AssetRow"));
 
     let row = chain_row
         .as_ref()
@@ -449,7 +449,7 @@ where
 {
     DBRow {
         key: asset_cache_key(asset_id),
-        value: bincode::serialize(&(stats, blockhash)).unwrap(),
+        value: bincode::serialize_little(&(stats, blockhash)).unwrap(),
     }
 }
 
@@ -492,7 +492,7 @@ where
         .store()
         .cache_db()
         .get(&asset_cache_key(asset_id))
-        .map(|c| bincode::deserialize(&c).unwrap())
+        .map(|c| bincode::deserialize_little(&c).unwrap())
         .and_then(|(stats, blockhash)| {
             chain
                 .height_by_hash(&blockhash)

diff --git a/src/new_index/db.rs b/src/new_index/db.rs
@@ -3,7 +3,7 @@ use rocksdb;
 use std::path::Path;
 
 use crate::config::Config;
-use crate::util::Bytes;
+use crate::util::{bincode, Bytes};
 
 static DB_VERSION: u32 = 1;
 
@@ -197,7 +197,7 @@ impl DB {
     }
 
     fn verify_compatibility(&self, config: &Config) {
-        let mut compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap();
+        let mut compatibility_bytes = bincode::serialize_little(&DB_VERSION).unwrap();
 
         if config.light_mode {
             // append a byte to indicate light_mode is enabled.

diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs
@@ -28,8 +28,8 @@ use crate::daemon::Daemon;
 use crate::errors::*;
 use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics};
 use crate::util::{
-    full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes,
-    HeaderEntry, HeaderList, ScriptToAddr,
+    bincode, full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta,
+    BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr,
 };
 
 use crate::new_index::db::{DBFlush, DBRow, ReverseScanIterator, ScanIterator, DB};
@@ -385,7 +385,7 @@ impl ChainQuery {
             self.store
                 .txstore_db
                 .get(&BlockRow::txids_key(full_hash(&hash[..])))
-                .map(|val| bincode::deserialize(&val).expect("failed to parse block txids"))
+                .map(|val| bincode::deserialize_little(&val).expect("failed to parse block txids"))
         }
     }
 
@@ -399,7 +399,7 @@ impl ChainQuery {
             self.store
                 .txstore_db
                 .get(&BlockRow::meta_key(full_hash(&hash[..])))
-                .map(|val| bincode::deserialize(&val).expect("failed to parse BlockMeta"))
+                .map(|val| bincode::deserialize_little(&val).expect("failed to parse BlockMeta"))
         }
     }
 
@@ -533,7 +533,7 @@ impl ChainQuery {
             .store
             .cache_db
             .get(&UtxoCacheRow::key(scripthash))
-            .map(|c| bincode::deserialize(&c).unwrap())
+            .map(|c| bincode::deserialize_little(&c).unwrap())
             .and_then(|(utxos_cache, blockhash)| {
                 self.height_by_hash(&blockhash)
                     .map(|height| (utxos_cache, height))
@@ -638,7 +638,7 @@ impl ChainQuery {
             .store
             .cache_db
             .get(&StatsCacheRow::key(scripthash))
-            .map(|c| bincode::deserialize(&c).unwrap())
+            .map(|c| bincode::deserialize_little(&c).unwrap())
             .and_then(|(stats, blockhash)| {
                 self.height_by_hash(&blockhash)
                     .map(|height| (stats, height))
@@ -1210,7 +1210,7 @@ impl TxRow {
     fn into_row(self) -> DBRow {
         let TxRow { key, value } = self;
         DBRow {
-            key: bincode::serialize(&key).unwrap(),
+            key: bincode::serialize_little(&key).unwrap(),
             value,
         }
     }
@@ -1245,14 +1245,14 @@ impl TxConfRow {
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: vec![],
         }
     }
 
     fn from_row(row: DBRow) -> Self {
         TxConfRow {
-            key: bincode::deserialize(&row.key).expect("failed to parse TxConfKey"),
+            key: bincode::deserialize_little(&row.key).expect("failed to parse TxConfKey"),
         }
     }
 }
@@ -1281,7 +1281,7 @@ impl TxOutRow {
         }
     }
     fn key(outpoint: &OutPoint) -> Bytes {
-        bincode::serialize(&TxOutKey {
+        bincode::serialize_little(&TxOutKey {
             code: b'O',
             txid: full_hash(&outpoint.txid[..]),
             vout: outpoint.vout as u16,
@@ -1291,7 +1291,7 @@ impl TxOutRow {
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: self.value,
         }
     }
@@ -1322,14 +1322,14 @@ impl BlockRow {
     fn new_txids(hash: FullHash, txids: &[Txid]) -> BlockRow {
         BlockRow {
             key: BlockKey { code: b'X', hash },
-            value: bincode::serialize(txids).unwrap(),
+            value: bincode::serialize_little(txids).unwrap(),
         }
     }
 
     fn new_meta(hash: FullHash, meta: &BlockMeta) -> BlockRow {
         BlockRow {
             key: BlockKey { code: b'M', hash },
-            value: bincode::serialize(meta).unwrap(),
+            value: bincode::serialize_little(meta).unwrap(),
         }
     }
 
@@ -1358,14 +1358,14 @@ impl BlockRow {
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: self.value,
         }
     }
 
     fn from_row(row: DBRow) -> Self {
         BlockRow {
-            key: bincode::deserialize(&row.key).unwrap(),
+            key: bincode::deserialize_little(&row.key).unwrap(),
             value: row.value,
         }
     }
@@ -1446,28 +1446,22 @@ impl TxHistoryRow {
     }
 
     fn prefix_end(code: u8, hash: &[u8]) -> Bytes {
-        bincode::serialize(&(code, full_hash(&hash[..]), std::u32::MAX)).unwrap()
+        bincode::serialize_big(&(code, full_hash(&hash[..]), std::u32::MAX)).unwrap()
     }
 
     fn prefix_height(code: u8, hash: &[u8], height: u32) -> Bytes {
-        bincode::config()
-            .big_endian()
-            .serialize(&(code, full_hash(&hash[..]), height))
-            .unwrap()
+        bincode::serialize_big(&(code, full_hash(&hash[..]), height)).unwrap()
     }
 
     pub fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::config().big_endian().serialize(&self.key).unwrap(),
+            key: bincode::serialize_big(&self.key).unwrap(),
             value: vec![],
         }
     }
 
     pub fn from_row(row: DBRow) -> Self {
-        let key = bincode::config()
-            .big_endian()
-            .deserialize(&row.key)
-            .expect("failed to deserialize TxHistoryKey");
+        let key = bincode::deserialize_big(&row.key).expect("failed to deserialize TxHistoryKey");
         TxHistoryRow { key }
     }
 
@@ -1533,19 +1527,20 @@ impl TxEdgeRow {
 
     fn filter(outpoint: &OutPoint) -> Bytes {
         // TODO build key without using bincode? [ b"S", &outpoint.txid[..], outpoint.vout?? ].concat()
-        bincode::serialize(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout as u16)).unwrap()
+        bincode::serialize_little(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout as u16))
+            .unwrap()
     }
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: vec![],
         }
     }
 
     fn from_row(row: DBRow) -> Self {
         TxEdgeRow {
-            key: bincode::deserialize(&row.key).expect("failed to deserialize TxEdgeKey"),
+            key: bincode::deserialize_little(&row.key).expect("failed to deserialize TxEdgeKey"),
         }
     }
 }
@@ -1568,7 +1563,7 @@ impl StatsCacheRow {
                 code: b'A',
                 scripthash: full_hash(scripthash),
             },
-            value: bincode::serialize(&(stats, blockhash)).unwrap(),
+            value: bincode::serialize_little(&(stats, blockhash)).unwrap(),
         }
     }
 
@@ -1578,7 +1573,7 @@ impl StatsCacheRow {
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: self.value,
         }
     }
@@ -1600,7 +1595,7 @@ impl UtxoCacheRow {
                 code: b'U',
                 scripthash: full_hash(scripthash),
             },
-            value: bincode::serialize(&(utxos_cache, blockhash)).unwrap(),
+            value: bincode::serialize_little(&(utxos_cache, blockhash)).unwrap(),
         }
     }
 
@@ -1610,7 +1605,7 @@ impl UtxoCacheRow {
 
     fn into_row(self) -> DBRow {
         DBRow {
-            key: bincode::serialize(&self.key).unwrap(),
+            key: bincode::serialize_little(&self.key).unwrap(),
             value: self.value,
         }
     }

diff --git a/src/util/bincode.rs b/src/util/bincode.rs
@@ -0,0 +1,66 @@
+//! This module creates two sets of serialize and deserialize for bincode.
+//! They explicitly spell out the bincode settings so that switching to
+//! new versions in the future is less error prone.
+//!
+//! This is a list of all the row types and their settings for bincode.
+//! +--------------+--------+------------+----------------+------------+
+//! |              | Endian | Int Length | Allow Trailing | Byte Limit |
+//! +--------------+--------+------------+----------------+------------+
+//! | TxHistoryRow | big    | fixed      | allow          | unlimited  |
+//! | All others   | little | fixed      | allow          | unlimited  |
+//! +--------------+--------+------------+----------------+------------+
+//!
+//! Based on @junderw's https://github.com/mempool/electrs/pull/34. Thanks!
+
+use bincode::Options;
+
+pub fn serialize_big<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    big_endian().serialize(value)
+}
+
+pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    big_endian().deserialize(bytes)
+}
+
+pub fn serialize_little<T>(value: &T) -> Result<Vec<u8>, bincode::Error>
+where
+    T: ?Sized + serde::Serialize,
+{
+    little_endian().serialize(value)
+}
+
+pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result<T, bincode::Error>
+where
+    T: serde::Deserialize<'a>,
+{
+    little_endian().deserialize(bytes)
+}
+
+/// This is the default settings for Options,
+/// but all explicitly spelled out, except for endianness.
+/// The following functions will add endianness.
+#[inline]
+fn options() -> impl Options {
+    bincode::options()
+        .with_fixint_encoding()
+        .with_no_limit()
+        .allow_trailing_bytes()
+}
+
+/// Adding the endian flag for big endian
+#[inline]
+fn big_endian() -> impl Options {
+    options().with_big_endian()
+}
+
+/// Adding the endian flag for little endian
+#[inline]
+fn little_endian() -> impl Options {
+    options().with_little_endian()
+}
diff --git a/src/util/mod.rs b/src/util/mod.rs
@@ -2,6 +2,7 @@ mod block;
 mod script;
 mod transaction;
 
+pub mod bincode;
 pub mod electrum_merkle;
 pub mod fees;