add more unit test for resending add tlc

Makefile

@@ -55,7 +55,7 @@ coverage: coverage-run-unittests coverage-collect-data coverage-generate-report
 
 .PHONY: gen-rpc-doc
 gen-rpc-doc:
-	cargo install fiber-rpc-gen
+	cargo binstall --no-confirm fiber-rpc-gen --force --locked
 	fiber-rpc-gen ./src/rpc
 	if grep -q "TODO: add desc" ./src/rpc/README.md; then \
         echo "Warning: There are 'TODO: add desc' in src/rpc/README.md, please add documentation comments to resolve them"; \

src/fiber/channel.rs

@@ -1,6 +1,6 @@
 #[cfg(debug_assertions)]
 use crate::fiber::network::DebugEvent;
-use crate::fiber::serde_utils::U64Hex;
+use crate::{debug_event, fiber::serde_utils::U64Hex};
 use bitflags::bitflags;
 use ckb_jsonrpc_types::BlockNumber;
 use futures::future::OptionFuture;
@@ -2097,16 +2097,7 @@ where
                         "Error while processing channel message: {:?} with message: {:?}",
                         error, message
                     );
-                    #[cfg(debug_assertions)]
-                    self.network
-                        .clone()
-                        .send_message(NetworkActorMessage::new_notification(
-                            NetworkServiceEvent::DebugEvent(DebugEvent::Common(format!(
-                                "{:?}",
-                                error
-                            ))),
-                        ))
-                        .expect(ASSUME_NETWORK_ACTOR_ALIVE);
+                    debug_event!(&self.network, &format!("{:?}", error));
                 }
             }
             ChannelActorMessage::Command(command) => {
@@ -5667,6 +5658,7 @@ impl ChannelActorState {
                             && matches!(info.outbound_status(), OutboundTlcStatus::LocalAnnounced)
                         {
                             // resend AddTlc message
+                            eprintln!("resending tlc ....");
                             network
                                 .send_message(NetworkActorMessage::new_command(
                                     NetworkActorCommand::SendFiberMessage(
@@ -5686,6 +5678,7 @@ impl ChannelActorState {
                                 ))
                                 .expect(ASSUME_NETWORK_ACTOR_ALIVE);
                             need_resend_commitment_signed = true;
+                            debug_event!(network, "resend add tlc");
                         } else if let Some(remove_reason) = &info.removed_reason {
                             if info.is_received()
                                 && matches!(info.inbound_status(), InboundTlcStatus::LocalRemoved)
@@ -5769,14 +5762,7 @@ impl ChannelActorState {
                     );
                 }
 
-                #[cfg(debug_assertions)]
-                network
-                    .send_message(NetworkActorMessage::new_notification(
-                        NetworkServiceEvent::DebugEvent(DebugEvent::Common(
-                            "Reestablished channel in ChannelReady".to_string(),
-                        )),
-                    ))
-                    .expect(ASSUME_NETWORK_ACTOR_ALIVE);
+                debug_event!(network, "Reestablished channel in ChannelReady");
             }
             _ => {
                 // TODO: @quake we need to handle other states.

src/fiber/network.rs

@@ -511,6 +511,18 @@ pub enum DebugEvent {
     Common(String),
 }
 
+#[macro_export]
+macro_rules! debug_event {
+    ($network:expr, $debug_event:expr) => {
+        #[cfg(debug_assertions)]
+        $network
+            .send_message(NetworkActorMessage::new_notification(
+                NetworkServiceEvent::DebugEvent(DebugEvent::Common($debug_event.to_string())),
+            ))
+            .expect(ASSUME_NETWORK_ACTOR_ALIVE);
+    };
+}
+
 #[derive(Clone, Debug)]
 pub enum NetworkServiceEvent {
     NetworkStarted(PeerId, MultiAddr, Vec<Multiaddr>),

src/fiber/tests/channel.rs

@@ -3785,6 +3785,76 @@ async fn test_connect_to_peers_with_mutual_channel_on_restart_2() {
     .await;
 }
 
+#[tokio::test]
+async fn test_send_payment_with_node_restart_then_resend_add_tlc() {
+    init_tracing();
+
+    let node_a_funding_amount = 100000000000;
+    let node_b_funding_amount = 6200000000;
+
+    let (mut node_a, mut node_b, _new_channel_id, _) =
+        NetworkNode::new_2_nodes_with_established_channel(
+            node_a_funding_amount,
+            node_b_funding_amount,
+            true,
+        )
+        .await;
+
+    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+
+    let node_b_pubkey = node_b.pubkey.clone();
+    let tlc_amount = 99;
+    let message = |rpc_reply| {
+        NetworkActorMessage::Command(NetworkActorCommand::SendPayment(
+            SendPaymentCommand {
+                target_pubkey: Some(node_b_pubkey),
+                amount: Some(tlc_amount),
+                payment_hash: None,
+                final_tlc_expiry_delta: None,
+                tlc_expiry_limit: None,
+                invoice: None,
+                timeout: None,
+                max_fee_amount: None,
+                max_parts: None,
+                keysend: Some(true),
+                udt_type_script: None,
+                allow_self_payment: false,
+                dry_run: false,
+            },
+            rpc_reply,
+        ))
+    };
+    let res = call!(node_a.network_actor, message).expect("node_a alive");
+    assert!(res.is_ok());
+    let payment_hash = res.unwrap().payment_hash;
+
+    node_b.stop().await;
+
+    let payment_status = node_a.get_payment_status(payment_hash).await;
+    assert_eq!(payment_status, PaymentSessionStatus::Inflight);
+
+    node_b.start().await;
+
+    node_a.expect_event(
+        |event| matches!(event, NetworkServiceEvent::PeerConnected(id, _addr) if id == &node_b.peer_id),
+    )
+    .await;
+
+    node_a
+        .expect_event(|event| {
+            matches!(
+            event,
+            NetworkServiceEvent::DebugEvent(DebugEvent::Common(
+                info
+            )) if "resend add tlc" == info)
+        })
+        .await;
+
+    tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
+    let payment_status = node_a.get_payment_status(payment_hash).await;
+    assert_eq!(payment_status, PaymentSessionStatus::Success);
+}
+
 #[tokio::test]
 async fn test_open_channel_with_large_size_shutdown_script_should_fail() {
     let [node_a, node_b] = NetworkNode::new_n_interconnected_nodes().await;

src/fiber/tests/test_utils.rs

@@ -555,18 +555,24 @@ impl NetworkNode {
         expected_status: PaymentSessionStatus,
         expected_retried: Option<u32>,
     ) {
+        let status = self.get_payment_status(payment_hash).await;
+        assert_eq!(status, expected_status);
+
+        if let Some(expected_retried) = expected_retried {
+            let payment_session = self.get_payment_session(payment_hash).unwrap();
+            assert_eq!(payment_session.retried_times, expected_retried);
+        }
+    }
+
+    pub async fn get_payment_status(&self, payment_hash: Hash256) -> PaymentSessionStatus {
         let message = |rpc_reply| -> NetworkActorMessage {
             NetworkActorMessage::Command(NetworkActorCommand::GetPayment(payment_hash, rpc_reply))
         };
         let res = call!(self.network_actor, message)
             .expect("node_a alive")
             .unwrap();
 
-        assert_eq!(res.status, expected_status);
-        if let Some(expected_retried) = expected_retried {
-            let payment_session = self.get_payment_session(payment_hash).unwrap();
-            assert_eq!(payment_session.retried_times, expected_retried);
-        }
+        res.status
     }
 
     pub async fn update_channel_actor_state(&mut self, state: ChannelActorState) {