From 7a547a5bfa15a044d2c80747e448fc82a10f1c50 Mon Sep 17 00:00:00 2001
From: AdityaAtulTewari <adityaatewari@gmail.com>
Date: Sat, 25 May 2024 17:36:26 -0500
Subject: [PATCH] Add schedule timing counters (#108)

---
 include/pando-lib-galois/loops/do_all.hpp | 24 +++++++++++++++++------
 pando-rt/src/init.cpp                     |  5 +++++
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/include/pando-lib-galois/loops/do_all.hpp b/include/pando-lib-galois/loops/do_all.hpp
index 7d1920c7..6993e654 100644
--- a/include/pando-lib-galois/loops/do_all.hpp
+++ b/include/pando-lib-galois/loops/do_all.hpp
@@ -17,8 +17,11 @@
 #include <pando-rt/memory/global_ptr.hpp>
 #include <pando-rt/pando-rt.hpp>
 
+constexpr bool SCHEDULER_TIMER_ENABLE = false;
+
 extern counter::Record<std::minstd_rand> perCoreRNG;
 extern counter::Record<std::uniform_int_distribution<std::int8_t>> perCoreDist;
+extern counter::Record<std::int64_t> schedulerCount;
 
 namespace galois {
 
@@ -47,8 +50,12 @@ enum SchedulerPolicy {
   RANDOM,
   UNSAFE_STRIPE,
   CORE_STRIPE,
+  NODE_ONLY,
 };
 
+constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = SchedulerPolicy::RANDOM;
+constexpr SchedulerPolicy EVENLY_PARITION_SCHEDULER_POLICY = SchedulerPolicy::CORE_STRIPE;
+
 template <enum SchedulerPolicy>
 struct LoopLocalSchedulerStruct {};
 
@@ -65,25 +72,31 @@ struct LoopLocalSchedulerStruct<SchedulerPolicy::CORE_STRIPE> {
 template <SchedulerPolicy Policy>
 pando::Place schedulerImpl(pando::Place preferredLocality,
                            [[maybe_unused]] LoopLocalSchedulerStruct<Policy>& loopLocal) noexcept {
+  counter::HighResolutionCount<SCHEDULER_TIMER_ENABLE> schedulerTimer;
+  schedulerTimer.start();
   if constexpr (Policy == RANDOM) {
     auto coreIdx = perCoreDist.getLocal()(perCoreRNG.getLocal());
     assert(coreIdx < pando::getCoreDims().x);
-    return pando::Place(preferredLocality.node, pando::anyPod, pando::CoreIndex(coreIdx, 0));
+    preferredLocality =
+        pando::Place(preferredLocality.node, pando::anyPod, pando::CoreIndex(coreIdx, 0));
   } else if constexpr (Policy == UNSAFE_STRIPE) {
     auto threadIdx = ++loopLocal.lastThreadIdx;
     threadIdx %= getNumThreads();
-    return std::get<0>(getPlaceFromThreadIdx(threadIdx));
+    preferredLocality = std::get<0>(getPlaceFromThreadIdx(threadIdx));
   } else if constexpr (Policy == CORE_STRIPE) {
     auto coreIdx = ++loopLocal.lastCoreIdx;
     coreIdx %= pando::getCoreDims().x;
-    return pando::Place(preferredLocality.node, pando::anyPod, pando::CoreIndex(coreIdx, 0));
+    preferredLocality =
+        pando::Place(preferredLocality.node, pando::anyPod, pando::CoreIndex(coreIdx, 0));
+  } else if constexpr (Policy == NODE_ONLY) {
+    preferredLocality = pando::Place(preferredLocality.node, pando::anyPod, pando::anyCore);
   } else {
     PANDO_ABORT("SCHEDULER POLICY NOT IMPLEMENTED");
   }
+  counter::recordHighResolutionEvent(schedulerCount, schedulerTimer);
+  return preferredLocality;
 }
 
-constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = SchedulerPolicy::RANDOM;
-
 inline pando::Place scheduler(pando::Place preferredLocality,
                               LoopLocalSchedulerStruct<CURRENT_SCHEDULER_POLICY>& loopLocal) {
   return schedulerImpl<CURRENT_SCHEDULER_POLICY>(preferredLocality, loopLocal);
@@ -357,7 +370,6 @@ class DoAll {
   template <typename State, typename F>
   static pando::Status doAllEvenlyPartition(WaitGroup::HandleType wgh, State s, uint64_t workItems,
                                             const F& func) {
-    constexpr SchedulerPolicy EVENLY_PARITION_SCHEDULER_POLICY = SchedulerPolicy::CORE_STRIPE;
     LoopLocalSchedulerStruct<EVENLY_PARITION_SCHEDULER_POLICY> loopLocal;
     pando::Status err = pando::Status::Success;
     if (workItems == 0) {
diff --git a/pando-rt/src/init.cpp b/pando-rt/src/init.cpp
index bc2e0969..5ddee8cf 100644
--- a/pando-rt/src/init.cpp
+++ b/pando-rt/src/init.cpp
@@ -34,6 +34,7 @@
 
 counter::Record<std::minstd_rand> perCoreRNG;
 counter::Record<std::uniform_int_distribution<std::int8_t>> perCoreDist;
+counter::Record<std::int64_t> schedulerCount = counter::Record<std::int64_t>();
 
 namespace pando {
 
@@ -211,6 +212,10 @@ int main(int argc, char* argv[]) {
         thisPlace.node.id,
         std::int8_t((i == std::uint64_t(dims.core.x + 1)) ? -1 : i),
         pointerCount.get(i));
+    SPDLOG_WARN("Scheduler time on node: {}, core: {} was {}",
+        thisPlace.node.id,
+        std::int8_t((i == std::uint64_t(dims.core.x + 1)) ? -1 : i),
+        schedulerCount.get(i));
   }