Add new REST API 'GET /mempool/txs[/:start_index]'

doc/API.md

@@ -208,6 +208,12 @@ Get a list of the last 10 transactions to enter the mempool.
 
 Each transaction object contains simplified overview data, with the following fields: `txid`, `fee`, `vsize`, `time` and `value`
 
+### `GET /mempool/txs[/:start_index]`
+
+Get a list of the transactions to enter the mempool (up to 25 transactions beginning at `start_index`).
+
+Each transaction object contains simplified overview data, with the following fields: `txid`, `fee`, `vsize`, `time` and `value`
+
 ## Fee estimates
 
 ### `GET /fee-estimates`

src/daemon.rs

@@ -15,8 +15,8 @@ use tapyrus::{BlockHash, Txid};
 use tapyrus::consensus::encode::{deserialize, serialize};
 
 use crate::chain::{Block, BlockHeader, Network, Transaction};
-use crate::new_index::mempool::MempoolTx;
 use crate::metrics::{HistogramOpts, HistogramVec, Metrics};
+use crate::new_index::mempool::MempoolTx;
 use crate::signal::Waiter;
 use crate::util::HeaderList;
 
@@ -501,10 +501,7 @@ impl Daemon {
     }
 
     pub fn getmempooltx(&self, txhash: &Txid) -> Result<MempoolTx> {
-        let res = self.request(
-            "getmempoolentry",
-            json!(txhash.to_hex()),
-        )?;
+        let res = self.request("getmempoolentry", json!(txhash.to_hex()))?;
         Ok(serde_json::from_value(res).chain_err(|| "invalid getmempoolentry reply")?)
     }
 

src/new_index/mempool.rs

@@ -38,6 +38,7 @@ pub struct Mempool {
     colors: HashMap<ColorIdentifier, Vec<ColoredTxHistoryInfo>>,
     edges: HashMap<OutPoint, (Txid, u32)>, // OutPoint -> (spending_txid, spending_vin)
     recent: ArrayDeque<[TxOverview; RECENT_TXS_SIZE], Wrapping>, // The N most recent txs to enter the mempool
+    overviews: HashMap<Txid, TxOverview>,
     backlog_stats: (BacklogStats, Instant),
 
     // monitoring
@@ -84,6 +85,7 @@ impl Mempool {
             colors: HashMap::new(),
             edges: HashMap::new(),
             recent: ArrayDeque::new(),
+            overviews: HashMap::new(),
             backlog_stats: (
                 BacklogStats::default(),
                 Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL),
@@ -229,7 +231,10 @@ impl Mempool {
         Ok(stats)
     }
 
-    pub fn get_colored_txs(&self, color_id: &ColorIdentifier) -> Vec<(Transaction, Option<BlockId>)> {
+    pub fn get_colored_txs(
+        &self,
+        color_id: &ColorIdentifier,
+    ) -> Vec<(Transaction, Option<BlockId>)> {
         let _timer = self
             .latency
             .with_label_values(&["get_colored_txs"])
@@ -238,7 +243,11 @@ impl Mempool {
             None => vec![],
             Some(entries) => entries
                 .iter()
-                .map(|info| self.txstore.get(&info.get_txid()).expect("missing mempool tx"))
+                .map(|info| {
+                    self.txstore
+                        .get(&info.get_txid())
+                        .expect("missing mempool tx")
+                })
                 .map(|tx| (tx.clone(), None))
                 .collect::<Vec<(Transaction, Option<BlockId>)>>(),
         };
@@ -259,6 +268,12 @@ impl Mempool {
         self.recent.iter().collect()
     }
 
+    pub fn txs_overview(&self) -> Vec<&TxOverview> {
+        let mut txs: Vec<&TxOverview> = self.overviews.values().collect();
+        txs.sort_by(|a, b| b.time.cmp(&a.time));
+        txs
+    }
+
     pub fn backlog_stats(&self) -> &BacklogStats {
         &self.backlog_stats.0
     }
@@ -282,10 +297,15 @@ impl Mempool {
                 return Ok(()); // keep the mempool until next update()
             }
         };
-        let to_add = to_add_tx.iter().map(|tx| {
-            let mempooltx = txs.get(&tx.malfix_txid()).expect("failed to get mempool tx");
-            (mempooltx.time, tx.clone())
-        }).collect();
+        let to_add = to_add_tx
+            .iter()
+            .map(|tx| {
+                let mempooltx = txs
+                    .get(&tx.malfix_txid())
+                    .expect("failed to get mempool tx");
+                (mempooltx.time, tx.clone())
+            })
+            .collect();
         // Add new transactions
         self.add(to_add);
         // Remove missing transactions
@@ -357,6 +377,17 @@ impl Mempool {
                 value: prevouts.values().map(|prevout| prevout.value).sum(),
             });
 
+            self.overviews.insert(
+                txid,
+                TxOverview {
+                    txid,
+                    fee: feeinfo.fee,
+                    vsize: feeinfo.vsize,
+                    time: *time,
+                    value: prevouts.values().map(|prevout| prevout.value).sum(),
+                },
+            );
+
             self.feeinfo.insert(txid, feeinfo);
 
             // An iterator over (ScriptHash, TxHistoryInfo)
@@ -507,6 +538,11 @@ impl Mempool {
                 .remove(*txid)
                 .unwrap_or_else(|| panic!("missing mempool tx {}", txid));
 
+            self.overviews.remove(*txid).or_else(|| {
+                warn!("missing mempool tx overviews {}", txid);
+                None
+            });
+
             self.feeinfo.remove(*txid).or_else(|| {
                 warn!("missing mempool tx feeinfo {}", txid);
                 None

src/new_index/query.rs

@@ -286,9 +286,17 @@ impl Query {
         )
     }
 
-    pub fn get_colored_txs(&self, color_id: &ColorIdentifier, last_seen_txid: Option<&Txid>, limit: usize,) -> Vec<(Transaction, Option<BlockId>)> {
+    pub fn get_colored_txs(
+        &self,
+        color_id: &ColorIdentifier,
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<(Transaction, Option<BlockId>)> {
         let mut txs = vec![];
-        txs.extend(self.chain().get_colored_txs(color_id, last_seen_txid, limit));
+        txs.extend(
+            self.chain()
+                .get_colored_txs(color_id, last_seen_txid, limit),
+        );
         txs.extend(self.mempool().get_colored_txs(color_id));
         txs
     }

src/new_index/schema.rs

@@ -444,7 +444,7 @@ impl ChainQuery {
 
     pub fn colored_history_iter_scan_reverse(
         &self,
-        color_id: &ColorIdentifier
+        color_id: &ColorIdentifier,
     ) -> ReverseScanIterator {
         self.store.history_db.iter_scan_reverse(
             &ColoredTxHistoryRow::filter(color_id),
@@ -718,7 +718,12 @@ impl ChainQuery {
         update_colored_stats(init_cache, &histories)
     }
 
-    pub fn get_colored_txs(&self, color_id: &ColorIdentifier, last_seen_txid: Option<&Txid>, limit: usize) -> Vec<(Transaction, Option<BlockId>)> {
+    pub fn get_colored_txs(
+        &self,
+        color_id: &ColorIdentifier,
+        last_seen_txid: Option<&Txid>,
+        limit: usize,
+    ) -> Vec<(Transaction, Option<BlockId>)> {
         let txids = self
             .colored_history_iter_scan_reverse(color_id)
             .map(|row| ColoredTxHistoryRow::from_row(row).get_txid())

src/rest.rs

@@ -2,6 +2,7 @@ use crate::chain::{address, Network, NetworkType, OutPoint, Transaction, TxIn, T
 use crate::config::Config;
 use crate::errors;
 use crate::new_index::color::ColoredStats;
+use crate::new_index::mempool::TxOverview;
 use crate::new_index::{compute_script_hash, Query, SpendingInput, Utxo};
 use crate::util::{
     create_socket, electrum_merkle, extract_tx_prevouts, full_hash, get_innerscripts,
@@ -34,6 +35,7 @@ use url::form_urlencoded;
 
 const CHAIN_TXS_PER_PAGE: usize = 25;
 const MAX_MEMPOOL_TXS: usize = 50;
+const MEMPOOL_TXS_PER_PAGE: usize = 25;
 const BLOCK_LIMIT: usize = 10;
 const ADDRESS_SEARCH_LIMIT: usize = 10;
 
@@ -865,6 +867,45 @@ fn handle_request(
             let recent = mempool.recent_txs_overview();
             json_response(recent, TTL_MEMPOOL_RECENT)
         }
+        (&Method::GET, Some(&"mempool"), Some(&"txs"), None, None, None) => {
+            let mempool = query.mempool();
+            let txs = mempool.txs_overview();
+            let total_num = txs.len();
+            let txs: Vec<&TxOverview> = txs.into_iter().take(MEMPOOL_TXS_PER_PAGE).collect();
+            let value = serde_json::to_string(&txs)?;
+            Ok(Response::builder()
+                .header("Content-Type", "application/json")
+                .header(
+                    "Cache-Control",
+                    format!("public, max-age={:}", TTL_MEMPOOL_RECENT),
+                )
+                .header("X-Total-Results", total_num.to_string())
+                .body(Body::from(value))
+                .unwrap())
+        }
+        (&Method::GET, Some(&"mempool"), Some(&"txs"), start_index, None, None) => {
+            let mempool = query.mempool();
+            let start_index = start_index
+                .map_or(0u32, |el| el.parse().unwrap_or(0))
+                .max(0u32) as usize;
+            let txs = mempool.txs_overview();
+            let total_num = txs.len();
+            let txs: Vec<&TxOverview> = txs
+                .into_iter()
+                .skip(start_index)
+                .take(MEMPOOL_TXS_PER_PAGE)
+                .collect();
+            let value = serde_json::to_string(&txs)?;
+            Ok(Response::builder()
+                .header("Content-Type", "application/json")
+                .header(
+                    "Cache-Control",
+                    format!("public, max-age={:}", TTL_MEMPOOL_RECENT),
+                )
+                .header("X-Total-Results", total_num.to_string())
+                .body(Body::from(value))
+                .unwrap())
+        }
 
         (&Method::GET, Some(&"color"), Some(color_id), None, None, None) => {
             let color_id = ColorIdentifier::from_hex(color_id).unwrap();
@@ -880,27 +921,20 @@ fn handle_request(
         (&Method::GET, Some(&"color"), Some(color_id), Some(&"txs"), None, None) => {
             let color_id = ColorIdentifier::from_hex(color_id).unwrap();
             let txs = query.get_colored_txs(&color_id, None, CHAIN_TXS_PER_PAGE);
-            json_response(
-                prepare_txs(
-                    txs,
-                    query,
-                    config,
-                ),
-                TTL_SHORT,
-            )
+            json_response(prepare_txs(txs, query, config), TTL_SHORT)
         }
-        (&Method::GET, Some(&"color"), Some(color_id), Some(&"txs"), Some(&"chain"), last_seen_txid) => {
+        (
+            &Method::GET,
+            Some(&"color"),
+            Some(color_id),
+            Some(&"txs"),
+            Some(&"chain"),
+            last_seen_txid,
+        ) => {
             let color_id = ColorIdentifier::from_hex(color_id).unwrap();
             let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_hex(txid).ok());
             let txs = query.get_colored_txs(&color_id, last_seen_txid.as_ref(), CHAIN_TXS_PER_PAGE);
-            json_response(
-                prepare_txs(
-                    txs,
-                    query,
-                    config,
-                ),
-                TTL_SHORT,
-            )
+            json_response(prepare_txs(txs, query, config), TTL_SHORT)
         }
 
         (&Method::GET, Some(&"fee-estimates"), None, None, None, None) => {