From 0739d50357d418a9f9194df5bce911da0e845dc8 Mon Sep 17 00:00:00 2001 From: Nadav Ivgi Date: Thu, 15 Feb 2024 11:57:27 +0200 Subject: [PATCH] Use bincode's new API, add utility functions for safer use This resolves the deprecation warnings for the old `bincode::config()`, and adds wrapper utility functions that explicitly spells out whether serialization should use little or big endianness. Switching from `bincode::config()` to `bincode::options()` changed the default settings, which requires explicitly enabling `with_fixint_encoding` and `allow_trailing_bytes` to restore the old behaviour. Thanks @junderw for the investigative work and writing the code this is based on (https://github.com/mempool/electrs/pull/34). --- src/bin/popular-scripts.rs | 8 ++--- src/elements/asset.rs | 10 +++--- src/new_index/db.rs | 4 +-- src/new_index/schema.rs | 59 ++++++++++++++++------------------ src/util/bincode.rs | 66 ++++++++++++++++++++++++++++++++++++++ src/util/mod.rs | 1 + 6 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 src/util/bincode.rs diff --git a/src/bin/popular-scripts.rs b/src/bin/popular-scripts.rs index 0f0976449..78b330628 100644 --- a/src/bin/popular-scripts.rs +++ b/src/bin/popular-scripts.rs @@ -1,9 +1,9 @@ extern crate electrs; -use bincode::Options; use electrs::{ config::Config, new_index::{Store, TxHistoryKey}, + util::bincode, }; use hex::DisplayHex; @@ -24,10 +24,8 @@ fn main() { break; } - let entry: TxHistoryKey = bincode::options() - .with_big_endian() - .deserialize(&key) - .expect("failed to deserialize TxHistoryKey"); + let entry: TxHistoryKey = + bincode::deserialize_big(&key).expect("failed to deserialize TxHistoryKey"); if curr_scripthash != entry.hash { if total_entries > 100 { diff --git a/src/elements/asset.rs b/src/elements/asset.rs index 8b88b3f5a..726431b54 100644 --- a/src/elements/asset.rs +++ b/src/elements/asset.rs @@ -13,7 +13,7 @@ use crate::elements::registry::{AssetMeta, AssetRegistry}; use crate::errors::*; use crate::new_index::schema::{TxHistoryInfo, TxHistoryKey, TxHistoryRow}; use crate::new_index::{db::DBFlush, ChainQuery, DBRow, Mempool, Query}; -use crate::util::{full_hash, Bytes, FullHash, TransactionStatus, TxInput}; +use crate::util::{bincode, full_hash, Bytes, FullHash, TransactionStatus, TxInput}; lazy_static! { pub static ref NATIVE_ASSET_ID: AssetId = @@ -192,7 +192,7 @@ pub fn index_confirmed_tx_assets( // reissuances are only kept under the history index. rows.extend(issuances.into_iter().map(|(asset_id, asset_row)| DBRow { key: [b"i", &asset_id.into_inner()[..]].concat(), - value: bincode::serialize(&asset_row).unwrap(), + value: bincode::serialize_little(&asset_row).unwrap(), })); } @@ -371,7 +371,7 @@ pub fn lookup_asset( let chain_row = history_db .get(&[b"i", &asset_id.into_inner()[..]].concat()) - .map(|row| bincode::deserialize::(&row).expect("failed parsing AssetRow")); + .map(|row| bincode::deserialize_little::(&row).expect("failed parsing AssetRow")); let row = chain_row .as_ref() @@ -449,7 +449,7 @@ where { DBRow { key: asset_cache_key(asset_id), - value: bincode::serialize(&(stats, blockhash)).unwrap(), + value: bincode::serialize_little(&(stats, blockhash)).unwrap(), } } @@ -492,7 +492,7 @@ where .store() .cache_db() .get(&asset_cache_key(asset_id)) - .map(|c| bincode::deserialize(&c).unwrap()) + .map(|c| bincode::deserialize_little(&c).unwrap()) .and_then(|(stats, blockhash)| { chain .height_by_hash(&blockhash) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index ef226d34b..be718680a 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -3,7 +3,7 @@ use rocksdb; use std::path::Path; use crate::config::Config; -use crate::util::Bytes; +use crate::util::{bincode, Bytes}; static DB_VERSION: u32 = 1; @@ -197,7 +197,7 @@ impl DB { } fn verify_compatibility(&self, config: &Config) { - let mut compatibility_bytes = bincode::serialize(&DB_VERSION).unwrap(); + let mut compatibility_bytes = bincode::serialize_little(&DB_VERSION).unwrap(); if config.light_mode { // append a byte to indicate light_mode is enabled. diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index f5afd1fb3..0641fb6e9 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -28,8 +28,8 @@ use crate::daemon::Daemon; use crate::errors::*; use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics}; use crate::util::{ - full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes, - HeaderEntry, HeaderList, ScriptToAddr, + bincode, full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, + BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr, }; use crate::new_index::db::{DBFlush, DBRow, ReverseScanIterator, ScanIterator, DB}; @@ -385,7 +385,7 @@ impl ChainQuery { self.store .txstore_db .get(&BlockRow::txids_key(full_hash(&hash[..]))) - .map(|val| bincode::deserialize(&val).expect("failed to parse block txids")) + .map(|val| bincode::deserialize_little(&val).expect("failed to parse block txids")) } } @@ -399,7 +399,7 @@ impl ChainQuery { self.store .txstore_db .get(&BlockRow::meta_key(full_hash(&hash[..]))) - .map(|val| bincode::deserialize(&val).expect("failed to parse BlockMeta")) + .map(|val| bincode::deserialize_little(&val).expect("failed to parse BlockMeta")) } } @@ -533,7 +533,7 @@ impl ChainQuery { .store .cache_db .get(&UtxoCacheRow::key(scripthash)) - .map(|c| bincode::deserialize(&c).unwrap()) + .map(|c| bincode::deserialize_little(&c).unwrap()) .and_then(|(utxos_cache, blockhash)| { self.height_by_hash(&blockhash) .map(|height| (utxos_cache, height)) @@ -638,7 +638,7 @@ impl ChainQuery { .store .cache_db .get(&StatsCacheRow::key(scripthash)) - .map(|c| bincode::deserialize(&c).unwrap()) + .map(|c| bincode::deserialize_little(&c).unwrap()) .and_then(|(stats, blockhash)| { self.height_by_hash(&blockhash) .map(|height| (stats, height)) @@ -1210,7 +1210,7 @@ impl TxRow { fn into_row(self) -> DBRow { let TxRow { key, value } = self; DBRow { - key: bincode::serialize(&key).unwrap(), + key: bincode::serialize_little(&key).unwrap(), value, } } @@ -1245,14 +1245,14 @@ impl TxConfRow { fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: vec![], } } fn from_row(row: DBRow) -> Self { TxConfRow { - key: bincode::deserialize(&row.key).expect("failed to parse TxConfKey"), + key: bincode::deserialize_little(&row.key).expect("failed to parse TxConfKey"), } } } @@ -1281,7 +1281,7 @@ impl TxOutRow { } } fn key(outpoint: &OutPoint) -> Bytes { - bincode::serialize(&TxOutKey { + bincode::serialize_little(&TxOutKey { code: b'O', txid: full_hash(&outpoint.txid[..]), vout: outpoint.vout as u16, @@ -1291,7 +1291,7 @@ impl TxOutRow { fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: self.value, } } @@ -1322,14 +1322,14 @@ impl BlockRow { fn new_txids(hash: FullHash, txids: &[Txid]) -> BlockRow { BlockRow { key: BlockKey { code: b'X', hash }, - value: bincode::serialize(txids).unwrap(), + value: bincode::serialize_little(txids).unwrap(), } } fn new_meta(hash: FullHash, meta: &BlockMeta) -> BlockRow { BlockRow { key: BlockKey { code: b'M', hash }, - value: bincode::serialize(meta).unwrap(), + value: bincode::serialize_little(meta).unwrap(), } } @@ -1358,14 +1358,14 @@ impl BlockRow { fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: self.value, } } fn from_row(row: DBRow) -> Self { BlockRow { - key: bincode::deserialize(&row.key).unwrap(), + key: bincode::deserialize_little(&row.key).unwrap(), value: row.value, } } @@ -1446,28 +1446,22 @@ impl TxHistoryRow { } fn prefix_end(code: u8, hash: &[u8]) -> Bytes { - bincode::serialize(&(code, full_hash(&hash[..]), std::u32::MAX)).unwrap() + bincode::serialize_big(&(code, full_hash(&hash[..]), std::u32::MAX)).unwrap() } fn prefix_height(code: u8, hash: &[u8], height: u32) -> Bytes { - bincode::config() - .big_endian() - .serialize(&(code, full_hash(&hash[..]), height)) - .unwrap() + bincode::serialize_big(&(code, full_hash(&hash[..]), height)).unwrap() } pub fn into_row(self) -> DBRow { DBRow { - key: bincode::config().big_endian().serialize(&self.key).unwrap(), + key: bincode::serialize_big(&self.key).unwrap(), value: vec![], } } pub fn from_row(row: DBRow) -> Self { - let key = bincode::config() - .big_endian() - .deserialize(&row.key) - .expect("failed to deserialize TxHistoryKey"); + let key = bincode::deserialize_big(&row.key).expect("failed to deserialize TxHistoryKey"); TxHistoryRow { key } } @@ -1533,19 +1527,20 @@ impl TxEdgeRow { fn filter(outpoint: &OutPoint) -> Bytes { // TODO build key without using bincode? [ b"S", &outpoint.txid[..], outpoint.vout?? ].concat() - bincode::serialize(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout as u16)).unwrap() + bincode::serialize_little(&(b'S', full_hash(&outpoint.txid[..]), outpoint.vout as u16)) + .unwrap() } fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: vec![], } } fn from_row(row: DBRow) -> Self { TxEdgeRow { - key: bincode::deserialize(&row.key).expect("failed to deserialize TxEdgeKey"), + key: bincode::deserialize_little(&row.key).expect("failed to deserialize TxEdgeKey"), } } } @@ -1568,7 +1563,7 @@ impl StatsCacheRow { code: b'A', scripthash: full_hash(scripthash), }, - value: bincode::serialize(&(stats, blockhash)).unwrap(), + value: bincode::serialize_little(&(stats, blockhash)).unwrap(), } } @@ -1578,7 +1573,7 @@ impl StatsCacheRow { fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: self.value, } } @@ -1600,7 +1595,7 @@ impl UtxoCacheRow { code: b'U', scripthash: full_hash(scripthash), }, - value: bincode::serialize(&(utxos_cache, blockhash)).unwrap(), + value: bincode::serialize_little(&(utxos_cache, blockhash)).unwrap(), } } @@ -1610,7 +1605,7 @@ impl UtxoCacheRow { fn into_row(self) -> DBRow { DBRow { - key: bincode::serialize(&self.key).unwrap(), + key: bincode::serialize_little(&self.key).unwrap(), value: self.value, } } diff --git a/src/util/bincode.rs b/src/util/bincode.rs new file mode 100644 index 000000000..d43a09189 --- /dev/null +++ b/src/util/bincode.rs @@ -0,0 +1,66 @@ +//! This module creates two sets of serialize and deserialize for bincode. +//! They explicitly spell out the bincode settings so that switching to +//! new versions in the future is less error prone. +//! +//! This is a list of all the row types and their settings for bincode. +//! +--------------+--------+------------+----------------+------------+ +//! | | Endian | Int Length | Allow Trailing | Byte Limit | +//! +--------------+--------+------------+----------------+------------+ +//! | TxHistoryRow | big | fixed | allow | unlimited | +//! | All others | little | fixed | allow | unlimited | +//! +--------------+--------+------------+----------------+------------+ +//! +//! Based on @junderw's https://github.com/mempool/electrs/pull/34. Thanks! + +use bincode::Options; + +pub fn serialize_big(value: &T) -> Result, bincode::Error> +where + T: ?Sized + serde::Serialize, +{ + big_endian().serialize(value) +} + +pub fn deserialize_big<'a, T>(bytes: &'a [u8]) -> Result +where + T: serde::Deserialize<'a>, +{ + big_endian().deserialize(bytes) +} + +pub fn serialize_little(value: &T) -> Result, bincode::Error> +where + T: ?Sized + serde::Serialize, +{ + little_endian().serialize(value) +} + +pub fn deserialize_little<'a, T>(bytes: &'a [u8]) -> Result +where + T: serde::Deserialize<'a>, +{ + little_endian().deserialize(bytes) +} + +/// This is the default settings for Options, +/// but all explicitly spelled out, except for endianness. +/// The following functions will add endianness. +#[inline] +fn options() -> impl Options { + bincode::options() + .with_fixint_encoding() + .with_no_limit() + .allow_trailing_bytes() +} + +/// Adding the endian flag for big endian +#[inline] +fn big_endian() -> impl Options { + options().with_big_endian() +} + +/// Adding the endian flag for little endian +#[inline] +fn little_endian() -> impl Options { + options().with_little_endian() +} diff --git a/src/util/mod.rs b/src/util/mod.rs index fbfaf7f97..3b03d41ce 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -2,6 +2,7 @@ mod block; mod script; mod transaction; +pub mod bincode; pub mod electrum_merkle; pub mod fees;