DrvX instrumentation for stat collection

.gitignore

@@ -44,7 +44,7 @@ logs/
 deps/
 
 #Tags Files
-tags
+*tags.csv
 
 #Stats Files
 *stats.csv

.pre-commit-config.yaml

@@ -54,7 +54,7 @@ repos:
     rev: v2.2.6
     hooks:
       - id: codespell
-        exclude: ^pando-drv/
+        exclude: ^pando-drv/|scripts/comparison.py
   - repo: https://github.com/pocc/pre-commit-hooks
     rev: v1.3.5
     hooks:

microbench/bfs/include/pando-bfs-galois/sssp.hpp

@@ -18,6 +18,7 @@
 #include <pando-lib-galois/containers/thread_local_vector.hpp>
 #include <pando-lib-galois/loops/do_all.hpp>
 #include <pando-rt/containers/vector.hpp>
+#include <pando-rt/drv_info.hpp>
 #include <pando-rt/memory/memory_guard.hpp>
 #include <pando-rt/sync/atomic.hpp>
 
@@ -139,6 +140,7 @@ pando::Status SSSP_DLCSR(
       }));
   PANDO_CHECK(wg.wait());
 #endif
+  PANDO_DRV_SET_STAGE_EXEC_COMP();
 
   while (!IsactiveIterationEmpty(phbfs)) {
 #ifdef DPRINTS
@@ -157,6 +159,8 @@ pando::Status SSSP_DLCSR(
     }
     PANDO_CHECK_RETURN(state.active.hostFlattenAppend(phbfs));
 
+    PANDO_DRV_INCREMENT_PHASE();
+
 #ifdef DPRINTS
     std::cerr << "Iteration loop end:\t" << state.dist - 1 << std::endl;
 #endif
@@ -169,6 +173,7 @@ pando::Status SSSP_DLCSR(
       }));
   PANDO_CHECK(wg.wait());
 #endif
+  PANDO_DRV_SET_STAGE_OTHER();
 
   if constexpr (COUNT_EDGE) {
     galois::doAll(
@@ -311,6 +316,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
       }));
   PANDO_CHECK(wg.wait());
 #endif
+  PANDO_DRV_SET_STAGE_EXEC_COMP();
 
   *active = true;
   while (*active) {
@@ -327,6 +333,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
         }));
     PANDO_CHECK_RETURN(wg.wait());
 
+    PANDO_DRV_SET_STAGE_EXEC_COMM();
     graph.template sync<decltype(updateData), true, false>(updateData);
 
     galois::HostLocalStorage<pando::Array<bool>> masterBitSets = graph.getMasterBitSets();
@@ -342,6 +349,8 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
 
     PANDO_CHECK_RETURN(wg.wait());
     graph.resetBitSets();
+    PANDO_DRV_SET_STAGE_EXEC_COMP();
+    PANDO_DRV_INCREMENT_PHASE();
 
 #ifdef DPRINTS
     std::cerr << "Iteration loop end:\t" << state.dist - 1 << std::endl;
@@ -355,6 +364,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
       }));
   PANDO_CHECK(wg.wait());
 #endif
+  PANDO_DRV_SET_STAGE_OTHER();
 
   if constexpr (COUNT_EDGE) {
     galois::doAll(

microbench/bfs/src/main.cpp

@@ -15,6 +15,7 @@
 #include <pando-lib-galois/graphs/mirror_dist_local_csr.hpp>
 #include <pando-lib-galois/import/ingest_rmat_el.hpp>
 #include <pando-rt/containers/vector.hpp>
+#include <pando-rt/drv_info.hpp>
 #include <pando-rt/memory/allocate_memory.hpp>
 #include <pando-rt/memory/memory_guard.hpp>
 #include <pando-rt/pando-rt.hpp>
@@ -36,6 +37,8 @@ void HBMainDLCSR(pando::Vector<std::uint64_t> srcVertices, std::uint64_t numVert
   using ET = std::uint64_t;
   using Graph = galois::DistLocalCSR<VT, ET>;
 
+  PANDO_DRV_SET_STAGE_INIT();
+
   Graph graph = galois::initializeELDLCSR<Graph, VT, ET>(filename, numVertices);
   filename.deinitialize();
 
@@ -84,6 +87,8 @@ void HBMainMDLCSR(pando::Vector<std::uint64_t> srcVertices, std::uint64_t numVer
   using ET = std::uint64_t;
   using Graph = galois::MirrorDistLocalCSR<VT, ET>;
 
+  PANDO_DRV_SET_STAGE_INIT();
+
   Graph graph = galois::initializeELDLCSR<Graph, VT, ET>(filename, numVertices);
   filename.deinitialize();
 

pando-drv/api/DrvAPIMemory.cpp

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2023 University of Washington
+
+#include <DrvAPIMemory.hpp>
+
+namespace DrvAPI
+{
+
+/**
+ * @brief set the stage
+ *
+ */
+void set_stage(stage_t stage)
+{
+    DrvAPIThread::current()->setState(std::make_shared<DrvAPISetStage>(stage));
+    DrvAPIThread::current()->yield();
+}
+
+/**
+ * @brief increment the phase
+ *
+ */
+void increment_phase()
+{
+    DrvAPIThread::current()->setState(std::make_shared<DrvAPIIncrementPhase>());
+    DrvAPIThread::current()->yield();
+}
+
+} // namespace DrvAPI

pando-drv/api/DrvAPIMemory.hpp

@@ -102,6 +102,18 @@ T atomic_cas(DrvAPIAddress address, T compare, T value)
     return result;
 }
 
+/**
+ * @brief set the stage
+ *
+ */
+extern void set_stage(stage_t stage);
+
+/**
+ * @brief increment the phase
+ *
+ */
+extern void increment_phase();
+
 /**
  * @brief memory fence
  *

pando-drv/api/DrvAPIThread.hpp

@@ -11,6 +11,7 @@
 #include <memory>
 namespace DrvAPI
 {
+
 class DrvAPIThread
 {
 public:

pando-drv/api/DrvAPIThreadState.hpp

@@ -10,6 +10,14 @@
 #include <stdlib.h>
 namespace DrvAPI
 {
+
+enum stage_t {
+    STAGE_INIT,
+    STAGE_EXEC_COMP,
+    STAGE_EXEC_COMM,
+    STAGE_OTHER
+};
+
 /**
  * @brief The thread state
  *
@@ -57,6 +65,34 @@ class DrvAPINop : public DrvAPIThreadState
     int  count_;
 };
 
+/**
+ * @brief Set stage thread state
+ */
+class DrvAPISetStage : public DrvAPIThreadState
+{
+public:
+    DrvAPISetStage(stage_t stage) : can_resume_(false), stage_(stage) {}
+    bool canResume() const override { return can_resume_; }
+    void complete(){ can_resume_ = true; }
+    stage_t getStage() const { return stage_; }
+private:
+    bool can_resume_;
+    stage_t  stage_;
+};
+
+/**
+ * @brief Increment Phase thread state
+ */
+class DrvAPIIncrementPhase : public DrvAPIThreadState
+{
+public:
+    DrvAPIIncrementPhase() : can_resume_(false) {}
+    bool canResume() const override { return can_resume_; }
+    void complete(){ can_resume_ = true; }
+private:
+    int can_resume_;
+};
+
 /**
  * @brief Base thread state for a memory operation
  *

pando-drv/api/Makefile

@@ -26,6 +26,7 @@ libdrvapi-sources += $(DRV_DIR)/api/DrvAPIThreadState.cpp
 libdrvapi-sources += $(DRV_DIR)/api/DrvAPIAddressToNative.cpp
 libdrvapi-sources += $(DRV_DIR)/api/DrvAPINativeToAddress.cpp
 libdrvapi-sources += $(DRV_DIR)/api/DrvAPIGlobal.cpp
+libdrvapi-sources += $(DRV_DIR)/api/DrvAPIMemory.cpp
 libdrvapi-objects := $(patsubst %.cpp,%.o,$(libdrvapi-sources))
 libdrvapi-headers := $(DRV_DIR)/api/DrvAPIAddress.hpp
 libdrvapi-headers += $(DRV_DIR)/api/DrvAPIAddressMap.hpp