Enable atomizer to wait until synched with watchtowers

Previously, the atomizer controller would make one attempt to connect to
watchtowers and would return false if the attempt failed.  This required
watchtowers to be initialized before atomizers, which required adding a
sleep to the atomizer end-to-end integration test so the watchtowers had time
to initialize.  This change enables the atomizer to make multiple attempts to
connect to watchtowers and precludes the need for the sleep in the
integration test.

Signed-off-by: Michael L. Szulczewski <[email protected]>

src/uhs/atomizer/atomizer/controller.cpp

@@ -59,10 +59,14 @@ namespace cbdc::atomizer {
     }
 
     auto controller::init() -> bool {
-        if(!m_watchtower_network.cluster_connect(
-               m_opts.m_watchtower_internal_endpoints)) {
-            m_logger->error("Failed to connect to watchtowers.");
-            return false;
+        static constexpr auto retry_delay = std::chrono::seconds(1);
+        m_watchtower_network.cluster_connect(
+            m_opts.m_watchtower_internal_endpoints,
+            false);
+        while(!m_watchtower_network.connected_to_one()) {
+            // TODO:  should we limit the number of attempts?
+            m_logger->warn("Failed to connect to any watchtowers, waiting...");
+            std::this_thread::sleep_for(retry_delay);
         }
 
         auto raft_params = nuraft::raft_params();

tests/integration/atomizer_end_to_end_test.cpp

@@ -47,7 +47,7 @@ class atomizer_end_to_end_test : public ::testing::Test {
             ASSERT_TRUE(m_ctl_watchtower->init());
         });
 
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        // std::this_thread::sleep_for(std::chrono::milliseconds(100));
 
         ASSERT_TRUE(m_ctl_atomizer->init());
         ASSERT_TRUE(m_ctl_archiver->init());