utcs-scea · AdityaAtulTewari · Jul 27, 2024
diff --git a/include/pando-lib-galois/containers/host_cached_array.hpp b/include/pando-lib-galois/containers/host_cached_array.hpp
@@ -55,7 +55,7 @@ class HostCachedArray {
     assert(range.size() == m_data.size());
     size_ = 0;
     PANDO_CHECK_RETURN(m_data.initialize());
-    PANDO_CHECK_RETURN(galois::doAll(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         range, m_data,
         +[](Range range, pando::GlobalRef<galois::HostIndexedMap<pando::Array<T>>> data) {
           PANDO_CHECK(lift(data, initialize));
@@ -64,7 +64,7 @@ class HostCachedArray {
               ref, initialize,
               *(range.begin() + static_cast<std::uint64_t>(pando::getCurrentPlace().node.id))));
         }));
-    PANDO_CHECK_RETURN(galois::doAll(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         m_data, m_data,
         +[](decltype(m_data) complete, galois::HostIndexedMap<pando::Array<T>> data) {
           for (std::uint64_t i = 0; i < data.size(); i++) {
@@ -79,7 +79,7 @@ class HostCachedArray {
   }
 
   void deinitialize() {
-    PANDO_CHECK(galois::doAll(
+    PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         m_data, +[](galois::HostIndexedMap<pando::Array<T>> data) {
           const std::uint64_t i = static_cast<std::uint64_t>(pando::getCurrentPlace().node.id);
           auto ref = data[i];

diff --git a/include/pando-lib-galois/containers/host_local_storage.hpp b/include/pando-lib-galois/containers/host_local_storage.hpp
@@ -287,7 +287,7 @@ template <typename T>
 [[nodiscard]] pando::Expected<galois::HostLocalStorage<T>> copyToAllHosts(T&& cont) {
   galois::HostLocalStorage<T> ret{};
   PANDO_CHECK_RETURN(ret.initialize());
-  PANDO_CHECK_RETURN(galois::doAll(
+  PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       cont, ret, +[](T cont, pando::GlobalRef<T> refcopy) {
         T copy;
         if (galois::localityOf(cont).node.id != pando::getCurrentPlace().node.id) {

diff --git a/include/pando-lib-galois/containers/pod_local_storage.hpp b/include/pando-lib-galois/containers/pod_local_storage.hpp
@@ -297,7 +297,7 @@ template <typename T>
 [[nodiscard]] pando::Expected<galois::PodLocalStorage<T>> copyToAllPods(T& cont) {
   galois::PodLocalStorage<T> ret{};
   PANDO_CHECK_RETURN(ret.initialize());
-  PANDO_CHECK_RETURN(galois::doAll(
+  PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       cont, ret, +[](T cont, pando::GlobalRef<T> refcopy) {
         T copy;
         const std::uint64_t size = cont.size();

diff --git a/include/pando-lib-galois/containers/thread_local_storage.hpp b/include/pando-lib-galois/containers/thread_local_storage.hpp
@@ -44,7 +44,7 @@ class ThreadLocalStorage {
 
   [[nodiscard]] pando::Status initialize() {
     PANDO_CHECK_RETURN(m_items.initialize());
-    PANDO_CHECK_RETURN(galois::doAll(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         m_items, +[](pando::GlobalRef<pando::GlobalPtr<T>> ref) {
           const auto placeDims = pando::getPlaceDims();
           const auto threadDims = pando::getThreadDims();
@@ -60,7 +60,7 @@ class ThreadLocalStorage {
   }
 
   void deinitialize() {
-    PANDO_CHECK(galois::doAll(
+    PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         m_items, +[](pando::GlobalRef<pando::GlobalPtr<T>> ptr) {
           const auto placeDims = pando::getPlaceDims();
           const auto threadDims = pando::getThreadDims();
@@ -275,7 +275,7 @@ template <typename T>
 [[nodiscard]] pando::Expected<galois::ThreadLocalStorage<T>> copyToAllThreads(T& cont) {
   galois::ThreadLocalStorage<T> ret{};
   PANDO_CHECK_RETURN(ret.initialize());
-  PANDO_CHECK_RETURN(galois::doAll(
+  PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       cont, ret, +[](T cont, pando::GlobalRef<T> refcopy) {
         T copy;
         const std::uint64_t size = cont.size();

diff --git a/include/pando-lib-galois/graphs/dist_local_csr.hpp b/include/pando-lib-galois/graphs/dist_local_csr.hpp
@@ -854,7 +854,7 @@ class DistLocalCSR {
     galois::HostLocalStorage<pando::Vector<VertexType>> pHV{};
     PANDO_CHECK_RETURN(pHV.initialize());
 
-    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         partEdges, pHV,
         +[](HostLocalStorage<pando::Vector<pando::Vector<EdgeType>>> partEdges,
             pando::GlobalRef<pando::Vector<VertexType>> pHV) {
@@ -1206,7 +1206,7 @@ class DistLocalCSR {
    * @brief create CSR Caches
    */
   pando::Status generateCache() {
-    return galois::doAll(
+    return galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         arrayOfCSRs, arrayOfCSRs,
         +[](decltype(arrayOfCSRs) arrayOfCSRs, HostIndexedMap<CSR> localCSRs) {
           for (std::uint64_t i = 0; i < localCSRs.size(); i++) {

diff --git a/include/pando-lib-galois/graphs/mirror_dist_local_csr.hpp b/include/pando-lib-galois/graphs/mirror_dist_local_csr.hpp
@@ -386,7 +386,7 @@ class MirrorDistLocalCSR {
    * @brief reset the master bit sets of all hosts
    */
   void resetMasterBitSets() {
-    galois::doAll(
+    galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         masterBitSets, +[](pando::GlobalRef<pando::Array<bool>> masterBitSet) {
           fmapVoid(masterBitSet, fill, false);
         });
@@ -395,7 +395,7 @@ class MirrorDistLocalCSR {
    * @brief reset the mirror bit sets of all hosts
    */
   void resetMirrorBitSets() {
-    galois::doAll(
+    galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         mirrorBitSets, +[](pando::GlobalRef<pando::Array<bool>> mirrorBitSet) {
           fmapVoid(mirrorBitSet, fill, false);
         });
@@ -576,7 +576,7 @@ class MirrorDistLocalCSR {
     auto thisMDLCSR = *this;
     auto state = galois::make_tpl(thisMDLCSR, func, wgh);
 
-    PANDO_CHECK(galois::doAll(
+    PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         wgh, state, localMirrorToRemoteMasterOrderedTable,
         +[](decltype(state) state,
             pando::Array<MirrorToMasterMap> localMirrorToRemoteMasterOrderedMap) {
@@ -636,7 +636,7 @@ class MirrorDistLocalCSR {
     auto thisMDLCSR = *this;
     auto state = galois::make_tpl(thisMDLCSR, wgh);
 
-    PANDO_CHECK(galois::doAll(
+    PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         wgh, state, localMasterToRemoteMirrorTable,
         +[](decltype(state) state,
             pando::Vector<pando::Vector<MirrorToMasterMap>> localMasterToRemoteMirrorMap) {
@@ -795,7 +795,7 @@ class MirrorDistLocalCSR {
     PANDO_CHECK_RETURN(mirrorBitSets.initialize());
     PANDO_CHECK_RETURN(masterBitSets.initialize());
     auto state = galois::make_tpl(masterRange, mirrorRange, mirrorBitSets);
-    PANDO_CHECK(galois::doAll(
+    PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         wgh, state, masterBitSets,
         +[](decltype(state) state, pando::GlobalRef<pando::Array<bool>> globalMasterBitSet) {
           auto [masterRange, mirrorRange, mirrorBitSets] = state;
@@ -839,7 +839,7 @@ class MirrorDistLocalCSR {
     // each host traverses its own localMirrorToRemoteMasterOrderedTable and send out the mapping to
     // the corresponding remote host append to the vector of vector where each vector is the mapping
     // from a specific host
-    galois::doAll(
+    galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
         state, localMirrorToRemoteMasterOrderedTable,
         +[](decltype(state) state,
             pando::Array<MirrorToMasterMap> localMirrorToRemoteMasterOrderedMap) {

diff --git a/include/pando-lib-galois/import/ingest_rmat_el.hpp b/include/pando-lib-galois/import/ingest_rmat_el.hpp
@@ -92,7 +92,7 @@ ReturnType initializeELDACSR(pando::Array<char> filename, std::uint64_t numVerti
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });
@@ -158,7 +158,7 @@ ReturnType initializeELDLCSR(pando::Array<char> filename, std::uint64_t numVerti
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });
@@ -204,16 +204,16 @@ ReturnType initializeELDLCSR(pando::Array<char> filename, std::uint64_t numVerti
     }
   };
 
-  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(generateVerticesState, pHV,
-                                                                   generateVerticesPerHost));
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
+      generateVerticesState, pHV, generateVerticesPerHost));
 
   auto [partEdges, renamePerHost] =
       internal::partitionEdgesParallely(pHV, std::move(localReadEdges), hostLocalV2PM);
 
-  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       partEdges, +[](pando::GlobalRef<pando::Vector<pando::Vector<ELEdge>>> edge_vectors) {
         pando::Vector<pando::Vector<ELEdge>> evs_tmp = edge_vectors;
-        galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+        galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
             evs_tmp, +[](pando::GlobalRef<pando::Vector<ELEdge>> src_ev) {
               pando::Vector<ELEdge> tmp = src_ev;
               std::sort(tmp.begin(), tmp.end());

diff --git a/include/pando-lib-galois/import/ingest_wmd_csv.hpp b/include/pando-lib-galois/import/ingest_wmd_csv.hpp
@@ -104,7 +104,7 @@ galois::DistLocalCSR<VertexType, EdgeType> initializeWMDDLCSR(pando::Array<char>
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });

diff --git a/include/pando-lib-galois/import/wmd_graph_importer.hpp b/include/pando-lib-galois/import/wmd_graph_importer.hpp
@@ -70,7 +70,7 @@ buildEdgeCountToSend(std::uint64_t numVirtualHosts,
     sumArray[i] = p;
   }
 
-  PANDO_CHECK_RETURN(galois::doAll(
+  PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       sumArray, localEdges,
       +[](pando::Array<galois::Pair<std::uint64_t, std::uint64_t>> counts,
           pando::Vector<pando::Vector<EdgeType>> localEdges) {
@@ -111,10 +111,10 @@ buildEdgeCountToSend(std::uint64_t numVirtualHosts,
   const uint64_t pairOffset = offsetof(UPair, first);
 
   auto state = galois::make_tpl(wgh, sumArray);
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       wgh, state, localEdges, +[](decltype(state) state, pando::Vector<EdgeType> localEdges) {
         auto [wgh, sumArray] = state;
-        PANDO_CHECK(galois::doAll(
+        PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
             wgh, sumArray, localEdges, +[](decltype(sumArray) counts, EdgeType localEdge) {
               pando::GlobalPtr<char> toAdd = static_cast<pando::GlobalPtr<char>>(
                   static_cast<pando::GlobalPtr<void>>(&counts[localEdge.src % counts.size()]));
@@ -179,7 +179,7 @@ partitionEdgesParallely(HostLocalStorage<pando::Vector<VertexType>> partitionedV
   // Insert into hashmap
   // ToDo: Parallelize this (Divija)
 
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       partitionedVertices, renamePerHost,
       +[](HostLocalStorage<pando::Vector<VertexType>> partitionedVertices,
           pando::GlobalRef<galois::HashTable<std::uint64_t, std::uint64_t>> hashRef) {
@@ -386,7 +386,7 @@ template <typename VertexType>
 
   auto tpl = galois::make_tpl(numVerticesPerHostPerThread, prefixArrPerHostPerThread,
                               perThreadVerticesPartition);
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       tpl, perThreadVerticesPartition,
       +[](decltype(tpl) tpl,
           pando::GlobalRef<HostIndexedMap<pando::Vector<VertexType>>> perThreadVerticesPartition) {
@@ -432,7 +432,7 @@ template <typename VertexType>
   using SRC_Val = uint64_t;
   using DST_Val = uint64_t;
 
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       numVerticesPerHostPerThread, prefixArrPerHostPerThread,
       +[](HostLocalStorage<Array<std::uint64_t>> numVerticesPerHostPerThread,
           Array<std::uint64_t> prefixArr) {
@@ -452,7 +452,7 @@ template <typename VertexType>
   WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize());
   auto freeWGH = freeWaiter.getHandle();
-  PANDO_CHECK(doAll(
+  PANDO_CHECK(doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       freeWGH, numVerticesPerHostPerThread, +[](galois::Array<std::uint64_t> arr) {
         arr.deinitialize();
       }));
@@ -461,7 +461,7 @@ template <typename VertexType>
   galois::HostLocalStorage<pando::Vector<VertexType>> pHV{};
   PANDO_CHECK(pHV.initialize());
 
-  PANDO_CHECK(doAll(
+  PANDO_CHECK(doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       pHV, prefixArrPerHostPerThread, +[](decltype(pHV) pHV, galois::Array<uint64_t> prefixArr) {
         const std::uint64_t nodeIdx = pando::getCurrentPlace().node.id;
         const std::uint64_t numThreads = galois::getNumThreads();
@@ -494,7 +494,7 @@ template <typename VertexType>
 
 #if FREE
   auto tpl = galois::make_tpl(prefixArrPerHostPerThread, perThreadVerticesPartition);
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       freeWGH, tpl, perThreadVerticesPartition,
       +[](decltype(tpl) tpl,
           pando::GlobalRef<HostIndexedMap<pando::Vector<VertexType>>> perThreadVerticesPartition) {

diff --git a/include/pando-lib-galois/loops/do_all.hpp b/include/pando-lib-galois/loops/do_all.hpp
@@ -22,6 +22,7 @@ constexpr bool DOALL_TIMER_ENABLE = false;
 
 extern counter::Record<std::minstd_rand> perCoreRNG;
 extern counter::Record<std::uniform_int_distribution<std::int8_t>> perCoreDist;
+extern counter::Record<std::uniform_int_distribution<std::int64_t>> nodeSizeDist;
 extern counter::Record<std::int64_t> schedulerCount;
 extern counter::Record<std::int64_t> doAllCount;
 
@@ -49,15 +50,16 @@ static inline uint64_t getTotalThreads() {
 }
 
 enum SchedulerPolicy {
-  RANDOM,
+  INFER_RANDOM_CORE,
   UNSAFE_STRIPE,
   CORE_STRIPE,
   NODE_ONLY,
+  NODE_RANDOM,
   NAIVE,
 };
 
 #ifndef PANDO_SCHED
-constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = SchedulerPolicy::RANDOM;
+constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = SchedulerPolicy::INFER_RANDOM_CORE;
 #else
 constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = PANDO_SCHED;
 #endif
@@ -82,7 +84,7 @@ pando::Place schedulerImpl(pando::Place preferredLocality,
                            [[maybe_unused]] LoopLocalSchedulerStruct<Policy>& loopLocal) noexcept {
   counter::HighResolutionCount<SCHEDULER_TIMER_ENABLE> schedulerTimer;
   schedulerTimer.start();
-  if constexpr (Policy == RANDOM) {
+  if constexpr (Policy == INFER_RANDOM_CORE) {
     auto coreIdx = perCoreDist.getLocal()(perCoreRNG.getLocal());
     assert(coreIdx < pando::getCoreDims().x);
     preferredLocality =
@@ -103,6 +105,10 @@ pando::Place schedulerImpl(pando::Place preferredLocality,
     assert(coreIdx < pando::getCoreDims().x);
     preferredLocality =
         pando::Place(pando::getCurrentNode(), pando::anyPod, pando::CoreIndex(coreIdx, 0));
+  } else if constexpr (Policy == NODE_RANDOM) {
+    auto nodeIdx = nodeSizeDist.getLocal()(perCoreRNG.getLocal());
+    assert(nodeIdx < pando::getNodeDims().id);
+    preferredLocality = pando::Place(pando::NodeIndex(nodeIdx), pando::anyPod, pando::anyCore);
   } else {
     PANDO_ABORT("SCHEDULER POLICY NOT IMPLEMENTED");
   }

diff --git a/microbench/bfs/include/pando-bfs-galois/sssp.hpp b/microbench/bfs/include/pando-bfs-galois/sssp.hpp
@@ -134,7 +134,7 @@ pando::Status SSSP_DLCSR(
   state.dist = 0;
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       wgh, phbfs, +[](pando::Vector<typename G::VertexTopologyID>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS Start");
       }));
@@ -168,7 +168,7 @@ pando::Status SSSP_DLCSR(
   }
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::INFER_RANDOM_CORE>(
       wgh, phbfs, +[](pando::Vector<typename G::VertexTopologyID>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS END");
       }));
@@ -311,7 +311,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
 #endif
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  PANDO_CHECK(galois::doAllExplicitPolicy<galois::SchedulerPolicy::INFER_RANDOM_CORE>(
       wgh, toRead, +[](MDWorkList<G>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS Start");
       }));
@@ -328,7 +328,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
     *active = false;
 
     auto state = galois::make_tpl(graph, toWrite);
-    PANDO_CHECK_RETURN(galois::doAll(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<galois::SchedulerPolicy::INFER_RANDOM_CORE>(
         wgh, state, toRead, +[](decltype(state) state, MDWorkList<G> toRead) {
           auto [graph, toWrite] = state;
           MDLCSRLocal<G>(graph, toRead, toWrite.getLocalRef());
@@ -360,7 +360,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
   }
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
+  PANDO_CHECK(galois::doAllExplicitPolicy<galois::SchedulerPolicy::INFER_RANDOM_CORE>(
       wgh, toRead, +[](MDWorkList<G>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS END");
       }));