Add missing failover params (#145)

add missing failover params It adds these failover parameters: 1) failover_timeout; 2) fencing_enabled; 3) fencing_timeout; 4) fencing_pause.
picodata · Nov 27, 2023 · ef58810 · ef58810
1 parent d74f1d1
commit ef58810
Show file tree

Hide file tree

Showing 10 changed files with 92 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -236,7 +236,7 @@ topology:
     replication_factor: 0     # (optional) number of replicas in replicaset, default for router 0
     weight: 10                # (optional) replicaset weight
     zone:                     # (optional) zone parameter for ansible cartridge playbook
-    roles:                    # (optional) list of roles    
+    roles:                    # (optional) list of roles
       - router
       - api
       - failover-coordinator
@@ -273,8 +273,13 @@ hosts:
 
 # failover parameters
 failover:
-  mode: stateful                      # (optional) failover mode (stateful, eventual, disabled)
-  state_provider: stateboard          # (optional) what is serve failover (stateboard, stateful)
+  mode: stateful               # (optional) failover mode (stateful, eventual, disabled)
+  state_provider: stateboard   # (optional) what is serve failover (stateboard, stateful)
+  failover_timeout: 60         # (optional) timeout (in seconds), used by membership to mark `suspect` members as `dead`
+  fencing_enabled: true        # (optional) abandon leadership when both the state provider quorum and at least one
+                               # replica are lost (suitable in stateful mode only)
+  fencing_timeout: 20          # (optional) time (in seconds) to actuate fencing after the check fails
+  fencing_pause: 120           # (optional) the period (in seconds) of performing the check
   stateboard_params:                  # (optional) params for chosen in state_provider failover type
       uri: 192.168.16.1:4401
       password: "vG?-GG!4sxV8q5:f"

diff --git a/README.ru.md b/README.ru.md
@@ -250,7 +250,7 @@ topolgy:
     replications_factor: 0    # (опционально) количество реплик в репликасете, для роутера по умолчанию 0
     weight: 10                # (опционально) вес репликасета (не учитывается генином)
     zone: host-1              # (опционально) параметр зоны для ansible cartridge
-    roles:                    # (опционально) список ролей в виде массива   
+    roles:                    # (опционально) список ролей в виде массива
       - router
       - api
       - failover-coordinator
@@ -290,9 +290,14 @@ hosts:
 
 # параметры фейловера
 failover:
-  mode: stateful                      # (опционально) вариант работы фейловера (stateful, eventual, disabled)
-  state_provider: stateboard          # (опционально) провайдер предоставляющий фейловер (stateboard, stateful)
-  stateboard_params:                  # (опционально) параметры для провайдера
+  mode: stateful              # (опционально) вариант работы фейловера (stateful, eventual, disabled)
+  state_provider: stateboard  # (опционально) провайдер предоставляющий фейловер (stateboard, stateful)
+  failover_timeout: 60        # (опционально) время (в секундах), определяет время перевода инстанса со статусом `suspect` в статус `dead`
+  fencing_enabled: true       # (опционально) если `true`, инстансы будут снимать с себя роль лидера при потере связи с провайдером состояния
+                              # и минимум одной репликой (работает только для stateful режима фейловера)
+  fencing_timeout: 20         # (опционально) время (в секундах) после которого лидер автоматически переходит в режим реплики
+  fencing_pause: 120          # (опционально) период (в секундах) между выполнением проверок fencing-алгоритма
+  stateboard_params:          # (опционально) параметры для провайдера
       uri:
         ip: 192.168.16.1
         port: 4401
@@ -707,7 +712,7 @@ genin upgrade --old cluster.genin.yml --new upgrade.genin.yml -o inventory.yml
 ```
 
 Опция `--old` указазывает путь к старому конфигу кластера которому мы хотим
-сделать `upgrade`.  
+сделать `upgrade`.
 Опция `--new` путь к новой конфигурации кластера, на основе которой `Genin`
 сделает `diff` и добавит те инстансы которых не было в конфиге переданном в
 `--old`.

diff --git a/src/task/cluster.rs b/src/task/cluster.rs
@@ -177,6 +177,7 @@ impl Default for Cluster {
                         password: String::from("password"),
                     },
                 ),
+                ..Default::default()
             },
             vars: Default::default(),
             metadata: ClusterMetadata {

diff --git a/src/task/cluster/test.rs b/src/task/cluster/test.rs
@@ -148,6 +148,7 @@ vars:
                 password: "some_password".into(),
             },
         ),
+        ..Default::default()
     });
 
     hosts_v2_model.spread();

diff --git a/src/task/flv.rs b/src/task/flv.rs
@@ -19,6 +19,10 @@ use super::{cluster::hst::v2::Address, AsError, TypeError, LIST, NUMBER, STRING}
 /// failover:
 ///     mode: stateful
 ///     state_provider: stateboard
+///     failover_timeout: 30
+///     fencing_enabled: false
+///     fencing_timeout: 30
+///     fencing_pause: 30
 ///     stateboard_params:
 ///         uri: "10.99.3.100:4001"
 ///         password: "vG?-GG!4sxV8q5:f"
@@ -27,6 +31,14 @@ pub struct Failover {
     pub mode: Mode,
     #[serde(skip_serializing_if = "StateProvider::is_disabled")]
     pub state_provider: StateProvider,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub failover_timeout: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fencing_enabled: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fencing_timeout: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fencing_pause: Option<usize>,
     #[serde(skip_serializing_if = "FailoverVariants::is_disabled", flatten)]
     pub failover_variants: FailoverVariants,
 }
@@ -37,6 +49,10 @@ impl Default for Failover {
             mode: Mode::Disabled,
             state_provider: StateProvider::Disabled,
             failover_variants: FailoverVariants::Disabled,
+            failover_timeout: None,
+            fencing_enabled: None,
+            fencing_timeout: None,
+            fencing_pause: None,
         }
     }
 }
@@ -54,6 +70,7 @@ impl<'a> TryFrom<&'a ArgMatches> for Failover {
                 mode: Mode::Disabled,
                 state_provider: StateProvider::Disabled,
                 failover_variants: FailoverVariants::Disabled,
+                ..Default::default()
             }),
             (_, Some("disabled")) => {
                 warn!(
@@ -65,17 +82,20 @@ impl<'a> TryFrom<&'a ArgMatches> for Failover {
                     mode: Mode::Disabled,
                     state_provider: StateProvider::Disabled,
                     failover_variants: FailoverVariants::Disabled,
+                ..Default::default()
                 })
             }
             (Some("eventual"), _) => Ok(Self {
                 mode: Mode::Eventual,
                 state_provider: StateProvider::Disabled,
                 failover_variants: FailoverVariants::Disabled,
+                ..Default::default()
             }),
             (Some("stateful"), Some(arg)) => Ok(Self {
                 mode: Mode::Stateful,
                 state_provider: StateProvider::try_from(arg)?,
                 failover_variants: FailoverVariants::try_from(arg)?,
+                ..Default::default()
             }),
             _ => Err(FailoverError::InvalidParams(
                 "Unknown failover options".into(),
@@ -109,6 +129,10 @@ impl<'de> Deserialize<'de> for Failover {
                 state_provider: StateProvider,
                 #[serde(flatten)]
                 failover_variants: FailoverVariants,
+                failover_timeout: Option<usize>,
+                fencing_enabled: Option<bool>,
+                fencing_timeout: Option<usize>,
+                fencing_pause: Option<usize>,
             },
             Disabled {
                 mode: Mode,
@@ -120,15 +144,27 @@ impl<'de> Deserialize<'de> for Failover {
                 mode,
                 state_provider,
                 failover_variants,
+                failover_timeout,
+                fencing_enabled,
+                fencing_timeout,
+                fencing_pause,
             }) => Ok(Self {
                 mode,
                 state_provider,
                 failover_variants,
+                failover_timeout,
+                fencing_enabled,
+                fencing_timeout,
+                fencing_pause,
             }),
             Ok(FailoverHelper::Disabled { mode }) => Ok(Self {
                 mode,
                 state_provider: StateProvider::Disabled,
                 failover_variants: FailoverVariants::Disabled,
+                failover_timeout: None,
+                fencing_enabled: None,
+                fencing_timeout: None,
+                fencing_pause: None,
             }),
             Err(e) => {
                 error!("Failover looks like {:?}", e);

diff --git a/src/task/flv/test.rs b/src/task/flv/test.rs
@@ -301,6 +301,7 @@ stateboard_params:
             },
             password: "some_password".to_string(),
         }),
+        ..Default::default()
     };
 
     assert_eq!(failover, failover_model);
@@ -312,8 +313,8 @@ fn failover_from_ip_and_port() {
 mode: stateful
 state_provider: stateboard
 stateboard_params:
-  uri: 
-    ip: 192.168.16.11 
+  uri:
+    ip: 192.168.16.11
     port: 4401
   password: some_password
 "#
@@ -331,6 +332,7 @@ stateboard_params:
             },
             password: "some_password".to_string(),
         }),
+        ..Default::default()
     };
 
     assert_eq!(failover, failover_model);
@@ -341,6 +343,10 @@ fn failover_to_str() {
     let failover = Failover {
         mode: Mode::Stateful,
         state_provider: StateProvider::Stateboard,
+        failover_timeout: Some(666),
+        fencing_enabled: Some(true),
+        fencing_pause: Some(42),
+        fencing_timeout: Some(24),
         failover_variants: FailoverVariants::StateboardVariant(StateboardParams {
             uri: Uri {
                 address: Address::Ip("192.168.16.11".parse().unwrap()),
@@ -353,6 +359,10 @@ fn failover_to_str() {
     let failover_model_str: String = r#"---
 mode: stateful
 state_provider: stateboard
+failover_timeout: 666
+fencing_enabled: true
+fencing_timeout: 24
+fencing_pause: 42
 stateboard_params:
   uri: "192.168.16.11:4401"
   password: some_password
@@ -387,6 +397,7 @@ stateboard_params:
             },
             password: "sosiska-123".into(),
         }),
+        ..Default::default()
     };
 
     assert_eq!(flv, flv_model);
@@ -400,10 +411,10 @@ etcd2_params:
   prefix: /cartridge
   lock_delay: 30
   endpoints:
-    - "http://172.20.73.12:2379" 
+    - "http://172.20.73.12:2379"
     - "http://172.20.73.13:2379"
     - "http://172.20.73.14:2379"
-  username: 111   
+  username: 111
   password: 111
 "#;
 
@@ -433,10 +444,10 @@ etcd2_params:
   prefix: /cartridge
   lock_delay: 30
   endpoints:
-    - "http://172.20.73.12:2379" 
+    - "http://172.20.73.12:2379"
     - "http://172.20.73.13:2379"
     - "http://172.20.73.14:2379"
-  username: 111   
+  username: 111
   password: 111
 "#;
 
@@ -488,10 +499,10 @@ etcd2_params:
   prefix: /cartridge
   lock_delay: 30
   endpoints:
-    - "http://172.20.73.12:2379" 
+    - "http://172.20.73.12:2379"
     - "http://172.20.73.13:2379"
     - "http://172.20.73.14:2379"
-  username: 111   
+  username: 111
   password: 111
 stateboard_params:
   uri: "192.168.16.11:4401"
@@ -512,9 +523,9 @@ etcd2_params:
   prefix: /cartridge
   lock_delay: hudred
   endpoints:
-    - "http://172.20.73.12:2379" 
+    - "http://172.20.73.12:2379"
     - 100000
-  username: 111   
+  username: 111
   password: 111
 "#;
 

diff --git a/src/task/vars.rs b/src/task/vars.rs
@@ -55,6 +55,7 @@ impl Default for Vars {
                 mode: Mode::Disabled,
                 state_provider: StateProvider::Disabled,
                 failover_variants: FailoverVariants::Disabled,
+                ..Default::default()
             }),
             another_fields: IndexMap::new(),
         }

diff --git a/src/task/vars/test.rs b/src/task/vars/test.rs
@@ -70,6 +70,7 @@ fn vars_failover() {
             },
             password: "some_password".to_string(),
         }),
+        ..Default::default()
     };
 
     let vars = Vars::from(&stateboard_failover);
@@ -85,6 +86,7 @@ fn vars_failover() {
                 },
                 password: "some_password".to_string(),
             }),
+            ..Default::default()
         }),
         ..Vars::default()
     };

diff --git a/tests/resources/cluster.genin.yml b/tests/resources/cluster.genin.yml
@@ -51,6 +51,14 @@ failover:
   mode: stateful
   # What is serve failover (stateboard, stateful)
   state_provider: stateboard
+  # Timeout (in seconds), used by membership to mark `suspect` members as `dead`
+  failover_timeout: 666
+  # Abandon leadership when both the state provider quorum and at least one replica are lost (suitable in stateful mode only)
+  fencing_enabled: true
+  # Time (in seconds) to actuate fencing after the check fails
+  fencing_timeout: 24
+  # The period (in seconds) of performing the check
+  fencing_pause: 42
   # Params for chosen in state_provider failover type
   stateboard_params:
     # Uri on which the stateboard will be available

diff --git a/tests/snapshots/r#mod__build_from_state.snap b/tests/snapshots/r#mod__build_from_state.snap
@@ -14,6 +14,10 @@ all:
     cartridge_failover_params:
       mode: stateful
       state_provider: stateboard
+      failover_timeout: 666
+      fencing_enabled: true
+      fencing_timeout: 24
+      fencing_pause: 42
       stateboard_params:
         uri: "192.168.16.11:4401"
         password: password
-Original file line number
+Diff line change
@@ Expand Up / @@ -148,6 +148,7 @@ vars: @@
                     password: "some_password".into(),
                 },
             ),
+            ..Default::default()
         });
         hosts_v2_model.spread();
@@ Expand Down @@