Added some quality of life features: more timers and options for elli…

…ptic-single.
DamynChipman · Feb 29, 2024 · 8a79581 · 8a79581
1 parent fc18564
commit 8a79581
Show file tree

Hide file tree

Showing 12 changed files with 168 additions and 37 deletions.
diff --git a/config.sh b/config.sh
@@ -11,7 +11,8 @@ MPI_PATH=/opt/homebrew
 P4EST_PATH=${HOME}/packages/p4est/p4est/build/local
 
 # PETSC_PATH : Path to PETSc install (i.e., ${PETSC_PATH}/include, ${PETSC_PATH}/lib, ...) 
-PETSC_PATH=${HOME}/packages/petsc/petsc-build-mumps
+# PETSC_PATH=${HOME}/packages/petsc/petsc-build-mumps
+PETSC_PATH=/opt/homebrew
 
 # BUILD_DIR : Build directory
 BUILD_DIR=build-$(git branch --show-current)

diff --git a/examples/elliptic-single/main.cpp b/examples/elliptic-single/main.cpp
@@ -127,12 +127,6 @@ int main(int argc, char** argv) {
     int n_solves = 1;
     app.options.setOption("n-solves", n_solves);
 
-    int min_level = 0;
-    app.options.setOption("min-level", min_level);
-
-    int max_level = 6;
-    app.options.setOption("max-level", max_level);
-
     double x_lower = -10.0;
     app.options.setOption("x-lower", x_lower);
 
@@ -145,10 +139,19 @@ int main(int argc, char** argv) {
     double y_upper = 10.0;
     app.options.setOption("y-upper", y_upper);
 
-    int nx = 8;
+    int min_level = 0;
+    int max_level = 5;
+    int nx = 16;
+    int ny = 16;
+    if (argc > 1) {
+        min_level = atoi(argv[1]);
+        max_level = atoi(argv[2]);
+        nx = atoi(argv[3]);
+        ny = atoi(argv[4]);
+    }
+    app.options.setOption("max-level", max_level);
+    app.options.setOption("min-level", min_level);
     app.options.setOption("nx", nx);
-
-    int ny = 8;
     app.options.setOption("ny", ny);
 
     // ====================================================

diff --git a/src/EllipticForestApp.cpp b/src/EllipticForestApp.cpp
@@ -2,12 +2,13 @@
 
 namespace EllipticForest {
 
-EllipticForestApp::EllipticForestApp() :
-    argc_(nullptr), argv_(nullptr)
-        {}
+EllipticForestApp::EllipticForestApp()
+    {}
 
 EllipticForestApp::EllipticForestApp(int* argc, char*** argv) :
-    argc_(argc), argv_(argv) {
+    argc_{argc},
+    argv_{argv},
+    parser{*argc, *argv} {
 
     addTimer("app-lifetime");
     timers["app-lifetime"].start();
@@ -19,14 +20,10 @@ EllipticForestApp::EllipticForestApp(int* argc, char*** argv) :
     PetscInitialize(argc_, argv_, NULL, NULL);
     PetscGetArgs(argc_, argv_);
 #endif
-
-    // Create options
-    InputParser inputParser(*argc_, *argv_);
-    inputParser.parse(options);
 
-    int myRank = -1;
-    MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
-    if (myRank == 0) {
+    int mpi_rank = -1;
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+    if (mpi_rank == 0) {
         std::cout << "[EllipticForest] Welcome to EllipticForest!" << std::endl;
     }
     this->actualClassPointer_ = this;
@@ -35,15 +32,31 @@ EllipticForestApp::EllipticForestApp(int* argc, char*** argv) :
 EllipticForestApp::~EllipticForestApp() {
     timers["app-lifetime"].stop();
 
-    int myRank = 0;
+    int mpi_rank = 0;
+    int mpi_size = 0;
     int isMPIFinalized;
     MPI_Finalized(&isMPIFinalized);
     if (!isMPIFinalized) {
-        MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+        MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+        MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
     }
 
-    if (myRank == 0) {
+    // std::map<std::string, std::vector<double>> global_timers;
+    // for (auto& [key, value] : timers) {
+    //     if (mpi_rank == 0) {
+    //         global_timers[key] = std::vector<double>(mpi_size);
+    //     }
+    //     double time = value.time();
+    //     MPI::gather<double>(time, global_timers[key], 0, MPI_COMM_WORLD);
+    // }
+
+    if (mpi_rank == 0) {
+        // std::string filename = "ef-timers-" + getCurrentDateTimeString() + ".csv";
+        // writeMapToCSV(global_timers, filename);
+
         std::cout << "[EllipticForest] End of app life cycle, finalizing..." << std::endl;
+        std::cout << "[EllipticForest] Options:" << std::endl << options;
+
         std::cout << "[EllipticForest] Timers: " << std::endl;
         for (auto& [key, value] : timers) {
             std::cout << "[EllipticForest]   " << key << " : " << value.time() << " [sec]" << std::endl;

diff --git a/src/EllipticForestApp.hpp b/src/EllipticForestApp.hpp
@@ -17,11 +17,13 @@
 #include <petsc.h>
 #include <p4est.h>
 
+#include "MPI.hpp"
 #include "Logger.hpp"
 #include "Options.hpp"
 #include "InputParser.hpp"
 #include "Timer.hpp"
 #include "GenericSingleton.hpp"
+#include "Helpers.hpp"
 
 namespace EllipticForest {
 
@@ -52,6 +54,8 @@ class EllipticForestApp : public GenericSingleton<EllipticForestApp> {
      */
     Options options{};
 
+    InputParser parser{};
+
     /**
      * @brief Timer instance for managing timers
      * 
@@ -126,13 +130,13 @@ class EllipticForestApp : public GenericSingleton<EllipticForestApp> {
      * @brief Pointer to argc from main
      * 
      */
-    int* argc_;
+    int* argc_ = nullptr;
 
     /**
      * @brief Pointer to argv from main
      * 
      */
-    char*** argv_;
+    char*** argv_ = nullptr;
 
 };
 

diff --git a/src/HPSAlgorithm.hpp b/src/HPSAlgorithm.hpp
@@ -123,10 +123,13 @@ class HPSAlgorithm : public MPI::MPIObject {
         EllipticForestApp& app = EllipticForestApp::getInstance();
         app.logHead("Begin HPS Build Stage");
         app.addTimer("build-stage");
-        app.timers["build-stage"].start();
+        app.addTimer("build-stage:leaf-callback");
+        app.addTimer("build-stage:family-callback");
 
+        app.timers["build-stage"].start();
         mesh.quadtree.merge(
             [&](Node<PatchType>* leaf_node){
+                app.timers["build-stage:leaf-callback"].start();
                 // app.log("Leaf callback: path = " + leaf_node->path);
                 // Leaf callback
                 PatchType& patch = leaf_node->data;
@@ -140,9 +143,12 @@ class HPSAlgorithm : public MPI::MPIObject {
                 else {
                     patch.matrixT() = patch_solver.buildD2N(patch.grid());
                 }
+
+                app.timers["build-stage:leaf-callback"].stop();
                 return 1;
             },
             [&](Node<PatchType>* parent_node, std::vector<Node<PatchType>*> child_nodes){
+                app.timers["build-stage:family-callback"].start();
                 // app.log("Family callback: parent path = " + parent_node->path);
                 // Family callback
                 PatchType& tau = parent_node->data;
@@ -151,11 +157,13 @@ class HPSAlgorithm : public MPI::MPIObject {
                 PatchType& gamma = child_nodes[2]->data;
                 PatchType& omega = child_nodes[3]->data;
                 merge4to1(tau, alpha, beta, gamma, omega);
+
+                app.timers["build-stage:family-callback"].stop();
                 return 1;
             }
         );
-
         app.timers["build-stage"].stop();
+
         app.logHead("End HPS Build Stage");
 
     }
@@ -181,10 +189,13 @@ class HPSAlgorithm : public MPI::MPIObject {
         EllipticForestApp& app = EllipticForestApp::getInstance();
         app.logHead("Begin HPS Upwards Stage");
         app.addTimer("upwards-stage");
-        app.timers["upwards-stage"].start();
+        app.addTimer("upwards-stage:leaf-callback");
+        app.addTimer("upwards-stage:family-callback");
 
+        app.timers["upwards-stage"].start();
         mesh.quadtree.merge(
             [&](Node<PatchType>* leaf_node){
+                app.timers["upwards-stage:leaf-callback"].start();
                 // app.log("Leaf callback: path = " + leaf_node->path);
                 // Leaf callback
                 PatchType& patch = leaf_node->data;
@@ -195,9 +206,11 @@ class HPSAlgorithm : public MPI::MPIObject {
                 // Set particular Neumann data using patch solver function
                 patch.vectorH() = patch_solver.particularNeumannData(patch.grid(), patch.vectorF());
 
+                app.timers["upwards-stage:leaf-callback"].stop();
                 return 1;
             },
             [&](Node<PatchType>* parent_node, std::vector<Node<PatchType>*> child_nodes){
+                app.timers["upwards-stage:family-callback"].start();
                 // app.log("Family callback: parent path = " + parent_node->path);
                 // Family callback
                 PatchType& tau = parent_node->data;
@@ -206,11 +219,12 @@ class HPSAlgorithm : public MPI::MPIObject {
                 PatchType& gamma = child_nodes[2]->data;
                 PatchType& omega = child_nodes[3]->data;
                 upwards4to1(tau, alpha, beta, gamma, omega);
+                app.timers["upwards-stage:family-callback"].stop();
                 return 1;
             }
         );
-
         app.timers["upwards-stage"].stop();
+
         app.logHead("End HPS Upwards Stage");
 
     }
@@ -228,10 +242,13 @@ class HPSAlgorithm : public MPI::MPIObject {
         EllipticForestApp& app = EllipticForestApp::getInstance();
         app.logHead("Begin HPS Upwards Stage");
         app.addTimer("upwards-stage");
-        app.timers["upwards-stage"].start();
+        app.addTimer("upwards-stage:leaf-callback");
+        app.addTimer("upwards-stage:family-callback");
 
+        app.timers["upwards-stage"].start();
         mesh.quadtree.merge(
             [&](Node<PatchType>* leaf_node){
+                app.timers["upwards-stage:leaf-callback"].start();
                 // app.log("Leaf callback: path = " + leaf_node->path);
                 // Leaf callback
                 PatchType& patch = leaf_node->data;
@@ -251,9 +268,11 @@ class HPSAlgorithm : public MPI::MPIObject {
                 // Set particular Neumann data using patch solver function
                 patch.vectorH() = patch_solver.particularNeumannData(grid, patch.vectorF());
 
+                app.timers["upwards-stage:leaf-callback"].stop();
                 return 1;
             },
             [&](Node<PatchType>* parent_node, std::vector<Node<PatchType>*> child_nodes){
+                app.timers["upwards-stage:family-callback"].start();
                 // app.log("Family callback: parent path = " + parent_node->path);
                 // Family callback
                 PatchType& tau = parent_node->data;
@@ -262,11 +281,12 @@ class HPSAlgorithm : public MPI::MPIObject {
                 PatchType& gamma = child_nodes[2]->data;
                 PatchType& omega = child_nodes[3]->data;
                 upwards4to1(tau, alpha, beta, gamma, omega);
+                app.timers["upwards-stage:family-callback"].stop();
                 return 1;
             }
         );
-
         app.timers["upwards-stage"].stop();
+
         app.logHead("End HPS Upwards Stage");
 
     }
@@ -293,19 +313,24 @@ class HPSAlgorithm : public MPI::MPIObject {
         EllipticForestApp& app = EllipticForestApp::getInstance();
         app.logHead("Begin HPS Solve Stage");
         app.addTimer("solve-stage");
-        app.timers["solve-stage"].start();
+        app.addTimer("solve-stage:leaf-callback");
+        app.addTimer("solve-stage:family-callback");
 
         // Set Dirichlet data on root patch
         boundary_data_function(mesh.quadtree.root());
 
+        app.timers["solve-stage"].start();
         mesh.quadtree.split(
             [&](Node<PatchType>* leaf_node){
+                app.timers["solve-stage:leaf-callback"].start();
                 // app.log("Leaf callback: path = " + leaf_node->path);
                 // Leaf callback
                 leafSolve(leaf_node->data);
+                app.timers["solve-stage:leaf-callback"].stop();
                 return 1;
             },
             [&](Node<PatchType>* parent_node, std::vector<Node<PatchType>*> child_nodes){
+                app.timers["solve-stage:family-callback"].start();
                 // app.log("Family callback: parent path = " + parent_node->path);
                 // Family callback
                 PatchType& tau = parent_node->data;
@@ -314,11 +339,12 @@ class HPSAlgorithm : public MPI::MPIObject {
                 PatchType& gamma = child_nodes[2]->data;
                 PatchType& omega = child_nodes[3]->data;
                 split1to4(tau, alpha, beta, gamma, omega);
+                app.timers["solve-stage:family-callback"].stop();
                 return 1;
             }
         );
-
         app.timers["solve-stage"].stop();
+
         app.logHead("End HPS Solve Stage");
 
     }
@@ -346,7 +372,8 @@ class HPSAlgorithm : public MPI::MPIObject {
         EllipticForestApp& app = EllipticForestApp::getInstance();
         app.logHead("Begin HPS Solve Stage");
         app.addTimer("solve-stage");
-        app.timers["solve-stage"].start();
+        app.addTimer("solve-stage:leaf-callback");
+        app.addTimer("solve-stage:family-callback");
 
         // Set up data for Dirichlet solve
         PatchType& rootPatch = mesh.quadtree.root();
@@ -419,14 +446,18 @@ class HPSAlgorithm : public MPI::MPIObject {
             g = solve(A, rhs);
         }
 
+        app.timers["solve-stage"].start();
         mesh.quadtree.split(
             [&](Node<PatchType>* leaf_node){
+                app.timers["solve-stage:leaf-callback"].start();
                 // app.log("Leaf callback: path = " + leaf_node->path);
                 // Leaf callback
                 leafSolve(leaf_node->data);
+                app.timers["solve-stage:leaf-callback"].stop();
                 return 1;
             },
             [&](Node<PatchType>* parent_node, std::vector<Node<PatchType>*> child_nodes){
+                app.timers["solve-stage:family-callback"].start();
                 // app.log("Family callback: parent path = " + parent_node->path);
                 // Family callback
                 PatchType& tau = parent_node->data;
@@ -435,11 +466,12 @@ class HPSAlgorithm : public MPI::MPIObject {
                 PatchType& gamma = child_nodes[2]->data;
                 PatchType& omega = child_nodes[3]->data;
                 split1to4(tau, alpha, beta, gamma, omega);
+                app.timers["solve-stage:family-callback"].stop();
                 return 1;
             }
         );
-
         app.timers["solve-stage"].stop();
+
         app.logHead("End HPS Solve Stage");
 
     }

diff --git a/src/Helpers.cpp b/src/Helpers.cpp
@@ -0,0 +1,33 @@
+#include "Helpers.hpp"
+
+namespace EllipticForest {
+
+std::string getCurrentDateTimeString() {
+    std::time_t now = std::time(nullptr);
+    std::tm timeinfo = *std::localtime(&now);
+
+    std::ostringstream oss;
+    oss << std::put_time(&timeinfo, "%Y%m%d%H%M%S");
+    return oss.str();
+}
+
+void writeMapToCSV(const std::map<std::string, std::vector<double>>& data, const std::string& filename) {
+    std::ofstream file(filename);
+    if (!file.is_open()) {
+        std::cerr << "Error opening file " << filename << std::endl;
+        return;
+    }
+
+    for (const auto& entry : data) {
+        file << entry.first; // Writing the key to the file
+        const auto& vec = entry.second;
+        for (const auto& value : vec) {
+            file << "," << value; // Writing values in the vector
+        }
+        file << std::endl; // End of row
+    }
+
+    file.close();
+}
+
+} // NAMESPACE : EllipticForest