Updates to Config file Auto-generation (includes #138)

CMakeLists.txt

@@ -76,5 +76,6 @@ endif()
 add_subdirectory(src)
 add_subdirectory(tests)
 add_subdirectory(benchmarks)
+add_subdirectory(tools/config_generator)
 add_subdirectory(tools/bench)
 add_subdirectory(tools/shard-seeder)

Dockerfile

@@ -52,7 +52,7 @@ COPY --from=builder /opt/tx-processor/scripts/test-transaction.sh ./scripts/test
 COPY --from=builder /opt/tx-processor/build/src/uhs/client/client-cli ./build/src/uhs/client/client-cli
 
 # Copy 2PC config
-COPY --from=builder /opt/tx-processor/2pc-compose.cfg ./2pc-compose.cfg
+COPY --from=builder /opt/tx-processor/config/general/2pc-compose.cfg ./config/general/2pc-compose.cfg
 
 # Create Atomizer Deployment Image
 FROM $IMAGE_VERSION AS atomizer
@@ -74,4 +74,4 @@ COPY --from=builder /opt/tx-processor/scripts/test-transaction.sh ./scripts/test
 COPY --from=builder /opt/tx-processor/build/src/uhs/client/client-cli ./build/src/uhs/client/client-cli
 
 # Copy atomizer config
-COPY --from=builder /opt/tx-processor/atomizer-compose.cfg ./atomizer-compose.cfg
+COPY --from=builder /opt/tx-processor/config/general/atomizer-compose.cfg ./config/general/atomizer-compose.cfg

README.md

@@ -160,38 +160,38 @@ You can find the images [in the Github Container Registry](https://github.com/mi
 ## Setup test wallets and test them
 
 The following commands are all performed from within the second container we started in the previous step.
-In each of the below commands, you should pass `atomizer-compose.cfg` instead of `2pc-compose.cfg` if you started the atomizer architecture.
+In each of the below commands, you should pass `config/general/atomizer-compose.cfg` instead of `config/general/2pc-compose.cfg` if you started the atomizer architecture.
 
 * Mint new coins (e.g., 10 new UTXOs each with a value of 5 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
   [2021-08-17 15:11:57.686] [WARN ] Existing wallet file not found
   [2021-08-17 15:11:57.686] [WARN ] Existing mempool not found
   4bc23da407c3a8110145c5b6c38199c8ec3b0e35ea66bbfd78f0ed65304ce6fa
   ```
 
   If using the atomizer architecture, you'll need to sync the wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Inspect the balance of a wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.50, UTXOs: 10, pending TXs: 0
   ```
 
 * Make a new wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat newaddress
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat newaddress
   [2021-08-17 15:13:16.148] [WARN ] Existing wallet file not found
   [2021-08-17 15:13:16.148] [WARN ] Existing mempool not found
   usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   ```
 
 * Send currency from the first wallet to the second wallet created in the previous step (e.g., 30 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   tx_id:
   cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e75
   Data for recipient importinput:
@@ -201,20 +201,20 @@ In each of the below commands, you should pass `atomizer-compose.cfg` instead of
 
   If using the atomizer architecture, you'll need to sync the sending wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Check that the currency is no longer available in the sending wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.20, UTXOs: 4, pending TXs: 0
   ```
 
 * Import coins to the receiving wallet using the string after `importinput` from the currency transfer step above
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat sync
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat sync
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat info
   Balance: $0.30, UTXOs: 1, pending TXs: 0
   ```
 

config/unit/2pc_template_unit_test.tmpl

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0

config/unit/atomizer_template_unit_test.tmpl

@@ -0,0 +1,32 @@
+2pc=0
+archiver_count=1
+atomizer_count=1
+shard_count=1
+sentinel_count=1
+watchtower_count=1
+target_block_interval=3000
+stxo_cache_depth=2
+target_block_interval=250
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_default_log_level="INFO"
+tmpl_sentinel_log_level="DEBUG"
+tmpl_shard_log_level="INFO"
+tmpl_watchtower_log_level="DEBUG"
+tmpl_archiver_log_level="DEBUG"
+tmpl_atomizer_log_level="DEBUG"

docker-compose-2pc.yml

@@ -9,7 +9,7 @@ services:
     image: opencbdc-tx-twophase
     tty: true
     restart: always
-    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc 2pc-compose.cfg 0
+    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc ./config/general/2pc-compose.cfg 0
     ports:
       - 5555:5555
     depends_on:
@@ -29,7 +29,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/coordinator/coordinatord 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/coordinator/coordinatord ./config/general/2pc-compose.cfg 0 0
     expose:
       - "7777"
     depends_on:
@@ -49,7 +49,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/locking_shard/locking-shardd 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/locking_shard/locking-shardd ./config/general/2pc-compose.cfg 0 0
     expose:
       - "6666"
     ports:

docker-compose-atomizer.yml

@@ -8,7 +8,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/watchtower/watchtowerd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/watchtower/watchtowerd ./config/general/atomizer-compose.cfg 0
     ports:
       - 8555:8555
     expose:
@@ -28,7 +28,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd ./config/general/atomizer-compose.cfg 0
     expose:
       - "5555"
     depends_on:
@@ -48,7 +48,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/archiver/archiverd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/archiver/archiverd ./config/general/atomizer-compose.cfg 0
     expose:
       - "4555"
     depends_on:
@@ -69,7 +69,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/shard/shardd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/shard/shardd ./config/general/atomizer-compose.cfg 0
     expose:
       - "6555"
     depends_on:
@@ -91,7 +91,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/sentinel/sentineld atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/sentinel/sentineld ./config/general/atomizer-compose.cfg 0
     ports:
       - 7555:7555
     depends_on:

scripts/test.sh

@@ -164,7 +164,7 @@ run_test_suite () {
 if [[ "$RUN_UNIT_TESTS" == "true" ]]
 then
     echo "Running unit tests..."
-    find "${REPO_TOP_DIR}"/tests/unit/ -name '*.cfg' \
+    find "${REPO_TOP_DIR}"/config/unit/ -name '*.cfg' \
         -exec rsync \{\} "$BUILD_DIR" \;
     run_test_suite "tests/unit/run_unit_tests" "unit_tests_coverage"
 else
@@ -175,7 +175,8 @@ echo
 if [[ "$RUN_INTEGRATION_TESTS" == "true" ]]
 then
     echo "Running integration tests..."
-    cp "${REPO_TOP_DIR}"/tests/integration/*.cfg "$BUILD_DIR"
+    cp "${REPO_TOP_DIR}"/config/integration/*.cfg "${BUILD_DIR}"
+    cp "${REPO_TOP_DIR}"/tools/config_generator/*.tmpl "${BUILD_DIR}"/tools/config_generator
     run_test_suite "tests/integration/run_integration_tests" \
         "integration_tests_coverage"
 else

tests/CMakeLists.txt

@@ -1,6 +1,6 @@
 project(tests)
 
-include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
+include_directories(. ../src ../ ../tools/config_generator ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
 set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
 
 add_library(util util.cpp)

tests/unit/CMakeLists.txt

@@ -5,6 +5,7 @@ add_executable(run_unit_tests archiver_test.cpp
                               atomizer_test.cpp
                               buffer_test.cpp
                               common/hash_test.cpp
+                              config_gen_test.cpp
                               config_test.cpp
                               coordinator/messages_test.cpp
                               locking_shard/format_test.cpp
@@ -50,6 +51,7 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY}
                                      transaction
                                      network
                                      common
+                                     config_generator
                                      serialization
                                      crypto
                                      secp256k1

tests/unit/config_gen_test.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2022 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+//                    MITRE Corporation
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "tools/config_generator/config_generator.hpp"
+#include "util/common/config.hpp"
+
+#include <filesystem>
+#include <gtest/gtest.h>
+#include <string>
+
+// TODO: Add file parsing tests.
+
+class config_generation_validation_test : public ::testing::Test {
+  protected:
+    void SetUp() override {
+        template_file_atomizer
+            = "../config/unit/atomizer_template_unit_test.tmpl";
+        template_file_2pc = "../config/unit/2pc_template_unit_test.tmpl";
+        port_num = 5555;
+    }
+
+    std::string template_file_atomizer;
+    std::string template_file_2pc;
+    size_t port_num;
+};
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_atomizer_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    cbdc::generate_config::config_generator new_config_gen(
+        template_file_atomizer,
+        port_num);
+    testing::internal::CaptureStderr();
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_NE(testing::internal::GetCapturedStderr().find("SUCCESS"), -1);
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_two_phase_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    cbdc::generate_config::config_generator new_config_gen(template_file_2pc,
+                                                           port_num);
+    testing::internal::CaptureStderr();
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_NE(testing::internal::GetCapturedStderr().find("SUCCESS"), -1);
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}

tools/config_generator/2pc_config_template.tmpl

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0