built first pass cw-snapshot-vector-map lib

Cargo.toml

@@ -92,6 +92,7 @@ cw-fund-distributor = { path = "./contracts/distribution/cw-fund-distributor", v
 cw-hooks = { path = "./packages/cw-hooks", version = "2.5.0" }
 cw-paginate-storage = { path = "./packages/cw-paginate-storage", version = "2.5.0" }
 cw-payroll-factory = { path = "./contracts/external/cw-payroll-factory", version = "2.5.0" }
+cw-snapshot-vector-map = { path = "./packages/cw-snapshot-vector-map", version = "2.5.0" }
 cw-stake-tracker = { path = "./packages/cw-stake-tracker", version = "2.5.0" }
 cw-tokenfactory-issuer = { path = "./contracts/external/cw-tokenfactory-issuer", version = "2.5.0", default-features = false }
 cw-tokenfactory-types = { path = "./packages/cw-tokenfactory-types", version = "2.5.0", default-features = false }

packages/cw-snapshot-vector-map/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "cw-snapshot-vector-map"
+authors = ["noah <[email protected]>"]
+description = "A CosmWasm vector map that allows reading the items that existed at any height in the past."
+edition = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+version = { workspace = true }
+
+[dependencies]
+cosmwasm-schema = { workspace = true }
+cosmwasm-std = { workspace = true }
+cw-storage-plus = { workspace = true }
+cw-utils = { workspace = true }
+cw20 = { workspace = true }
+serde = { workspace = true }

packages/cw-snapshot-vector-map/README.md

@@ -0,0 +1,125 @@
+# CW Snapshot Vector Map
+
+A snapshot vector map maps keys to vectors of items, where the vectors' sets of
+items can be read at any height in the past. Items can be given an expiration,
+after which they automatically disappear from the vector. This minimizes
+redundant storage while allowing for efficient querying of historical data on a
+changing set of items.
+
+Because this uses a `SnapshotMap` under the hood, it's important to note that
+all pushes and removals occuring on a given block will be reflected on the
+following block. Since expirations are computed relative to the block they are
+pushed at, an expiration of 1 block means the item will never appear in the
+vector. More concretely, if an item is pushed at block `n` with an expiration of
+`m`, it will be included in the vector when queried at block `n + 1` up to `n +
+m - 1`. The vector at block `n + m` will no longer include the item.
+
+## Performance
+
+All operations (push/remove/load) run in O(n). When pushing/removing, `n` refers
+to the number of items in the most recent version of the vector. When loading,
+`n` refers to the number of items in the vector at the given block.
+
+Storage is optimized by only storing each pushed item once, referencing them in
+snapshots by numeric IDs that are much more compact. IDs are duplicated when the
+vector is changed, while items are never duplicated.
+
+The default `load` function can paginate item loading, but it first requires
+loading the entire set of IDs from storage. Thus there is some maximum number of
+items that can be stored based on gas limits and storage costs. However, this
+capacity is greatly increased by snapshotting IDs rather than items directly.
+
+## Limitations
+
+This data structure is only designed to be updated in the present and read in
+the past/present. More concretely, items can only be pushed or removed at a
+block greater than or equal to the last block at which an item was pushed or
+removed.
+
+Since all IDs must be loaded from storage before paginating item loading, there
+is a maximum number of items that can be stored based on gas limits and storage
+costs. This will vary by chain configuration but is likely quite high due to the
+compact ID storage.
+
+## Example
+
+```rust
+use cosmwasm_std::{testing::mock_dependencies, Addr, BlockInfo, Timestamp};
+use cw20::Expiration;
+use cw_utils::Duration;
+use cw_snapshot_vector_map::{LoadedItem, SnapshotVectorMap};
+
+macro_rules! b {
+    ($x:expr) => {
+        &BlockInfo {
+            chain_id: "CHAIN".to_string(),
+            height: $x,
+            time: Timestamp::from_seconds($x),
+        }
+    };
+}
+
+let storage = &mut mock_dependencies().storage;
+let svm: SnapshotVectorMap<Addr, String> = SnapshotVectorMap::new(
+    "svm__items",
+    "svm__next_ids",
+    "svm__active",
+    "svm__active__checkpoints",
+    "svm__active__changelog",
+);
+let key = Addr::unchecked("leaf");
+let first = "first".to_string();
+let second = "second".to_string();
+
+// store the first item at block 1, expiring in 10 blocks (at block 11)
+svm.push(storage, &key, &first, b!(1), Some(Duration::Height(10))).unwrap();
+
+// store the second item at block 5, which does not expire
+svm.push(storage, &key, &second, b!(5), None).unwrap();
+
+// remove the second item (ID: 1) at height 15
+svm.remove(storage, &key, 1, b!(15)).unwrap();
+
+// the vector at block 3 should contain only the first item
+assert_eq!(
+    svm.load_all(storage, &key, b!(3)).unwrap(),
+    vec![LoadedItem {
+        id: 0,
+        item: first.clone(),
+        expiration: Some(Expiration::AtHeight(11)),
+    }]
+);
+
+// the vector at block 7 should contain both items
+assert_eq!(
+    svm.load_all(storage, &key, b!(7)).unwrap(),
+    vec![
+        LoadedItem {
+            id: 0,
+            item: first.clone(),
+            expiration: Some(Expiration::AtHeight(11)),
+        },
+        LoadedItem {
+            id: 1,
+            item: second.clone(),
+            expiration: None,
+        }
+    ]
+);
+
+// the vector at block 12 should contain only the first item
+assert_eq!(
+    svm.load_all(storage, &key, b!(12)).unwrap(),
+    vec![LoadedItem {
+        id: 1,
+        item: second.clone(),
+        expiration: None,
+    }]
+);
+
+// the vector at block 17 should contain nothing
+assert_eq!(
+    svm.load_all(storage, &key, b!(17)).unwrap(),
+    vec![]
+);
+```

packages/cw-snapshot-vector-map/src/lib.rs

@@ -0,0 +1,207 @@
+#![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))]
+
+use cw20::Expiration;
+use cw_utils::Duration;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use cosmwasm_std::{BlockInfo, StdResult, Storage};
+use cw_storage_plus::{KeyDeserialize, Map, Prefixer, PrimaryKey, SnapshotMap, Strategy};
+
+/// Map to a vector that allows reading the subset of items that existed at a
+/// specific height in the past based on when items were added, removed, and
+/// expired.
+pub struct SnapshotVectorMap<'a, K, V> {
+    /// All items for a key, indexed by ID.
+    items: Map<'a, &'a (K, u64), V>,
+    /// The next item ID to use per-key.
+    next_ids: Map<'a, K, u64>,
+    /// The IDs of the items that are active for a key at a given height, and
+    /// optionally when they expire.
+    active: SnapshotMap<'a, K, Vec<(u64, Option<Expiration>)>>,
+}
+
+/// A loaded item from the vector, including its ID and expiration.
+#[derive(Debug, Clone, PartialEq)]
+pub struct LoadedItem<V> {
+    /// The ID of the item within the vector, which can be used to update or
+    /// remove it.
+    pub id: u64,
+    /// The item.
+    pub item: V,
+    /// When the item expires, if set.
+    pub expiration: Option<Expiration>,
+}
+
+impl<'a, K, V> SnapshotVectorMap<'a, K, V> {
+    /// Creates a new [`SnapshotVectorMap`] with the given storage keys.
+    ///
+    /// Example:
+    ///
+    /// ```rust
+    /// use cw_snapshot_vector_map::SnapshotVectorMap;
+    ///
+    /// SnapshotVectorMap::<&[u8], &str>::new(
+    ///     "data__items",
+    ///     "data__next_ids",
+    ///     "data__active",
+    ///     "data__active__checkpoints",
+    ///     "data__active__changelog",
+    /// );
+    /// ```
+    pub const fn new(
+        items_key: &'static str,
+        next_ids_key: &'static str,
+        active_key: &'static str,
+        active_checkpoints_key: &'static str,
+        active_changelog_key: &'static str,
+    ) -> Self {
+        SnapshotVectorMap {
+            items: Map::new(items_key),
+            next_ids: Map::new(next_ids_key),
+            active: SnapshotMap::new(
+                active_key,
+                active_checkpoints_key,
+                active_changelog_key,
+                Strategy::EveryBlock,
+            ),
+        }
+    }
+}
+
+impl<'a, K, V> SnapshotVectorMap<'a, K, V>
+where
+    // values can be serialized and deserialized
+    V: Serialize + DeserializeOwned,
+    // keys can be primary keys, cloned, deserialized, and prefixed
+    K: Clone + KeyDeserialize + Prefixer<'a> + PrimaryKey<'a>,
+    // &(key, ID) is a key in a map
+    for<'b> &'b (K, u64): PrimaryKey<'b>,
+{
+    /// Adds an item to the vector at the current block, optionally expiring in
+    /// the future, returning the ID of the new item. This block should be
+    /// greater than or equal to the blocks all previous items were
+    /// added/removed at. Pushing to the past will lead to incorrect behavior.
+    pub fn push(
+        &self,
+        store: &mut dyn Storage,
+        k: &K,
+        data: &V,
+        block: &BlockInfo,
+        expire_in: Option<Duration>,
+    ) -> StdResult<u64> {
+        // get next ID for the key, defaulting to 0
+        let next_id = self
+            .next_ids
+            .may_load(store, k.clone())?
+            .unwrap_or_default();
+
+        // add item to the list of all items for the key
+        self.items.save(store, &(k.clone(), next_id), data)?;
+
+        // get active list for the key
+        let mut active = self.active.may_load(store, k.clone())?.unwrap_or_default();
+
+        // remove expired items
+        active.retain(|(_, expiration)| {
+            expiration.map_or(true, |expiration| !expiration.is_expired(block))
+        });
+
+        // add new item and save list
+        active.push((next_id, expire_in.map(|d| d.after(block))));
+
+        // save the new list
+        self.active.save(store, k.clone(), &active, block.height)?;
+
+        // update next ID
+        self.next_ids.save(store, k.clone(), &(next_id + 1))?;
+
+        Ok(next_id)
+    }
+
+    /// Removes an item from the vector by ID and returns it. The block should
+    /// be greater than or equal to the blocks all previous items were
+    /// added/removed at. Removing from the past will lead to incorrect
+    /// behavior.
+    pub fn remove(
+        &self,
+        store: &mut dyn Storage,
+        k: &K,
+        id: u64,
+        block: &BlockInfo,
+    ) -> StdResult<V> {
+        // get active list for the key
+        let mut active = self.active.may_load(store, k.clone())?.unwrap_or_default();
+
+        // remove item and any expired items
+        active.retain(|(active_id, expiration)| {
+            active_id != &id && expiration.map_or(true, |expiration| !expiration.is_expired(block))
+        });
+
+        // save the new list
+        self.active.save(store, k.clone(), &active, block.height)?;
+
+        // load and return the item
+        self.load_item(store, k, id)
+    }
+
+    /// Loads paged items at the given block that are not expired.
+    pub fn load(
+        &self,
+        store: &dyn Storage,
+        k: &K,
+        block: &BlockInfo,
+        limit: Option<u64>,
+        offset: Option<u64>,
+    ) -> StdResult<Vec<LoadedItem<V>>> {
+        let offset = offset.unwrap_or_default() as usize;
+        let limit = limit.unwrap_or(u64::MAX) as usize;
+
+        let active_ids = self
+            .active
+            .may_load_at_height(store, k.clone(), block.height)?
+            .unwrap_or_default();
+
+        // load paged items, skipping expired ones
+        let items = active_ids
+            .iter()
+            .filter(|(_, expiration)| expiration.map_or(true, |exp| !exp.is_expired(block)))
+            .skip(offset)
+            .take(limit)
+            .map(|(id, expiration)| -> StdResult<LoadedItem<V>> {
+                let item = self.load_item(store, k, *id)?;
+                Ok(LoadedItem {
+                    id: *id,
+                    item,
+                    expiration: *expiration,
+                })
+            })
+            .collect::<StdResult<Vec<_>>>()?;
+
+        Ok(items)
+    }
+
+    /// Loads all items at the given block that are not expired.
+    pub fn load_all(
+        &self,
+        store: &dyn Storage,
+        k: &K,
+        block: &BlockInfo,
+    ) -> StdResult<Vec<LoadedItem<V>>> {
+        self.load(store, k, block, None, None)
+    }
+
+    /// Loads an item from the vector by ID.
+    pub fn load_item(&self, store: &dyn Storage, k: &K, id: u64) -> StdResult<V> {
+        let item = self.items.load(store, &(k.clone(), id))?;
+        Ok(item)
+    }
+
+    /// Loads an item from the vector by ID, if it exists.
+    pub fn may_load_item(&self, store: &dyn Storage, k: &K, id: u64) -> StdResult<Option<V>> {
+        self.items.may_load(store, &(k.clone(), id))
+    }
+}
+
+#[cfg(test)]
+mod tests;