From f203454b60b8af1fd06c653a76ac434b5b9755b0 Mon Sep 17 00:00:00 2001 From: AyanRoy16 Date: Wed, 11 Oct 2023 10:18:58 -0400 Subject: [PATCH 1/7] Cache line size declared in JCpuInfo and imported in the other files --- src/libraries/JANA/Engine/JMailbox.h | 3 ++- src/libraries/JANA/Engine/JSubeventMailbox.h | 3 ++- src/libraries/JANA/Utils/JCpuInfo.h | 2 ++ src/libraries/JANA/Utils/JEventPool.h | 4 ++-- src/libraries/JANA/Utils/JResourcePool.h | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libraries/JANA/Engine/JMailbox.h b/src/libraries/JANA/Engine/JMailbox.h index e2a13030f..e252444dd 100644 --- a/src/libraries/JANA/Engine/JMailbox.h +++ b/src/libraries/JANA/Engine/JMailbox.h @@ -8,6 +8,7 @@ #include #include +#include #include /// JMailbox is a threadsafe event queue designed for communication between Arrows. @@ -33,7 +34,7 @@ #ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES 64 +#define CACHE_LINE_BYTES JCpuInfo::getCacheLineByte #endif diff --git a/src/libraries/JANA/Engine/JSubeventMailbox.h b/src/libraries/JANA/Engine/JSubeventMailbox.h index b82e783ad..d8bd524b2 100644 --- a/src/libraries/JANA/Engine/JSubeventMailbox.h +++ b/src/libraries/JANA/Engine/JSubeventMailbox.h @@ -10,9 +10,10 @@ #include #include #include +#include #ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES 64 +#define CACHE_LINE_BYTES JCpuInfo::getCacheLineByte #endif diff --git a/src/libraries/JANA/Utils/JCpuInfo.h b/src/libraries/JANA/Utils/JCpuInfo.h index c47de2a7e..d2663e655 100644 --- a/src/libraries/JANA/Utils/JCpuInfo.h +++ b/src/libraries/JANA/Utils/JCpuInfo.h @@ -8,6 +8,8 @@ namespace JCpuInfo { + const uint32_t getCacheLineByte = 64; + size_t GetNumCpus(); uint32_t GetCpuID(); diff --git a/src/libraries/JANA/Utils/JEventPool.h b/src/libraries/JANA/Utils/JEventPool.h index df81cced8..86e7912c4 100644 --- a/src/libraries/JANA/Utils/JEventPool.h +++ b/src/libraries/JANA/Utils/JEventPool.h @@ -7,13 +7,13 @@ #define JANA2_JEVENTPOOL_H #include +#include #include #include - class JEventPool { private: - struct alignas(64) LocalPool { + struct alignas(JCpuInfo::getCacheLineByte) LocalPool { std::mutex mutex; std::vector> events; }; diff --git a/src/libraries/JANA/Utils/JResourcePool.h b/src/libraries/JANA/Utils/JResourcePool.h index 4023c2c4b..65b63f6e0 100644 --- a/src/libraries/JANA/Utils/JResourcePool.h +++ b/src/libraries/JANA/Utils/JResourcePool.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -98,7 +99,7 @@ template class JResourcePool //cache line size is 64 for ifarm1402, gcc won't allow larger than 128 //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size - return 64; //units are in bytes + return JCpuInfo::getCacheLineByte ; //units are in bytes } private: From 31f4f71bdb3e287df17977ac3ce09cf483dd9689 Mon Sep 17 00:00:00 2001 From: AyanRoy16 Date: Thu, 12 Oct 2023 08:53:35 -0400 Subject: [PATCH 2/7] Changing the cache lize size variable name --- src/libraries/JANA/Engine/JMailbox.h | 2 +- src/libraries/JANA/Engine/JSubeventMailbox.h | 2 +- src/libraries/JANA/Utils/JCpuInfo.h | 2 +- src/libraries/JANA/Utils/JEventPool.h | 2 +- src/libraries/JANA/Utils/JResourcePool.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libraries/JANA/Engine/JMailbox.h b/src/libraries/JANA/Engine/JMailbox.h index e252444dd..6a5a04ca9 100644 --- a/src/libraries/JANA/Engine/JMailbox.h +++ b/src/libraries/JANA/Engine/JMailbox.h @@ -34,7 +34,7 @@ #ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES JCpuInfo::getCacheLineByte +#define CACHE_LINE_BYTES JCpuInfo::JANA2_CACHE_LINE_BYTES #endif diff --git a/src/libraries/JANA/Engine/JSubeventMailbox.h b/src/libraries/JANA/Engine/JSubeventMailbox.h index d8bd524b2..518b7ddc9 100644 --- a/src/libraries/JANA/Engine/JSubeventMailbox.h +++ b/src/libraries/JANA/Engine/JSubeventMailbox.h @@ -13,7 +13,7 @@ #include #ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES JCpuInfo::getCacheLineByte +#define CACHE_LINE_BYTES JCpuInfo::JANA2_CACHE_LINE_BYTES #endif diff --git a/src/libraries/JANA/Utils/JCpuInfo.h b/src/libraries/JANA/Utils/JCpuInfo.h index d2663e655..826bb0210 100644 --- a/src/libraries/JANA/Utils/JCpuInfo.h +++ b/src/libraries/JANA/Utils/JCpuInfo.h @@ -8,7 +8,7 @@ namespace JCpuInfo { - const uint32_t getCacheLineByte = 64; + const uint32_t JANA2_CACHE_LINE_BYTES = 64; size_t GetNumCpus(); diff --git a/src/libraries/JANA/Utils/JEventPool.h b/src/libraries/JANA/Utils/JEventPool.h index 86e7912c4..d431efbed 100644 --- a/src/libraries/JANA/Utils/JEventPool.h +++ b/src/libraries/JANA/Utils/JEventPool.h @@ -13,7 +13,7 @@ class JEventPool { private: - struct alignas(JCpuInfo::getCacheLineByte) LocalPool { + struct alignas(JCpuInfo::JANA2_CACHE_LINE_BYTES) LocalPool { std::mutex mutex; std::vector> events; }; diff --git a/src/libraries/JANA/Utils/JResourcePool.h b/src/libraries/JANA/Utils/JResourcePool.h index 65b63f6e0..42be754e7 100644 --- a/src/libraries/JANA/Utils/JResourcePool.h +++ b/src/libraries/JANA/Utils/JResourcePool.h @@ -99,7 +99,7 @@ template class JResourcePool //cache line size is 64 for ifarm1402, gcc won't allow larger than 128 //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size - return JCpuInfo::getCacheLineByte ; //units are in bytes + return JCpuInfo::JANA2_CACHE_LINE_BYTES ; //units are in bytes } private: From e2f15897c0beb3143b22b5fab961ddff7e7e99b6 Mon Sep 17 00:00:00 2001 From: AyanRoy16 Date: Thu, 12 Oct 2023 10:44:37 -0400 Subject: [PATCH 3/7] Sophisticated Cache limit size change --- src/libraries/JANA/Engine/JMailbox.h | 5 ----- src/libraries/JANA/Engine/JSubeventMailbox.h | 7 +------ src/libraries/JANA/Utils/JCpuInfo.h | 7 ++++--- src/libraries/JANA/Utils/JEventPool.h | 2 +- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/libraries/JANA/Engine/JMailbox.h b/src/libraries/JANA/Engine/JMailbox.h index 6a5a04ca9..2ca6432e1 100644 --- a/src/libraries/JANA/Engine/JMailbox.h +++ b/src/libraries/JANA/Engine/JMailbox.h @@ -33,11 +33,6 @@ /// 3. Triple mutex trick to give push() priority? -#ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES JCpuInfo::JANA2_CACHE_LINE_BYTES -#endif - - template class JMailbox { diff --git a/src/libraries/JANA/Engine/JSubeventMailbox.h b/src/libraries/JANA/Engine/JSubeventMailbox.h index 518b7ddc9..fe9a2f6ec 100644 --- a/src/libraries/JANA/Engine/JSubeventMailbox.h +++ b/src/libraries/JANA/Engine/JSubeventMailbox.h @@ -12,11 +12,6 @@ #include #include -#ifndef CACHE_LINE_BYTES -#define CACHE_LINE_BYTES JCpuInfo::JANA2_CACHE_LINE_BYTES -#endif - - template struct SubeventWrapper { @@ -38,7 +33,7 @@ class JSubeventMailbox { private: - struct alignas(CACHE_LINE_BYTES) LocalMailbox { + struct alignas(JANA2_CACHE_LINE_BYTES) LocalMailbox { std::mutex mutex; std::deque> ready; std::map, size_t> in_progress; diff --git a/src/libraries/JANA/Utils/JCpuInfo.h b/src/libraries/JANA/Utils/JCpuInfo.h index 826bb0210..83b5d504b 100644 --- a/src/libraries/JANA/Utils/JCpuInfo.h +++ b/src/libraries/JANA/Utils/JCpuInfo.h @@ -1,15 +1,16 @@ // Copyright 2020, Jefferson Science Associates, LLC. // Subject to the terms in the LICENSE file found in the top-level directory. - #pragma once +#ifndef JANA2_CACHE_LINE_BYTES +#define JANA2_CACHE_LINE_BYTES 64 +#endif + #include namespace JCpuInfo { - const uint32_t JANA2_CACHE_LINE_BYTES = 64; - size_t GetNumCpus(); uint32_t GetCpuID(); diff --git a/src/libraries/JANA/Utils/JEventPool.h b/src/libraries/JANA/Utils/JEventPool.h index d431efbed..890d2968c 100644 --- a/src/libraries/JANA/Utils/JEventPool.h +++ b/src/libraries/JANA/Utils/JEventPool.h @@ -13,7 +13,7 @@ class JEventPool { private: - struct alignas(JCpuInfo::JANA2_CACHE_LINE_BYTES) LocalPool { + struct alignas(JANA2_CACHE_LINE_BYTES) LocalPool { std::mutex mutex; std::vector> events; }; From a7044811bcee72f10c5953c9064d74e74fa317a0 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 12 Oct 2023 11:05:44 -0400 Subject: [PATCH 4/7] Migrate JResourcePool to use JANA2_CACHE_LINE_BYTES --- src/libraries/JANA/Utils/JResourcePool.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libraries/JANA/Utils/JResourcePool.h b/src/libraries/JANA/Utils/JResourcePool.h index 42be754e7..d9df91bc3 100644 --- a/src/libraries/JANA/Utils/JResourcePool.h +++ b/src/libraries/JANA/Utils/JResourcePool.h @@ -99,7 +99,7 @@ template class JResourcePool //cache line size is 64 for ifarm1402, gcc won't allow larger than 128 //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size - return JCpuInfo::JANA2_CACHE_LINE_BYTES ; //units are in bytes + return JANA2_CACHE_LINE_BYTES ; //units are in bytes } private: @@ -120,10 +120,10 @@ template class JResourcePool void Recycle_Resources_StaticPool(std::vector& sResources); void Recycle_Resource_StaticPool(DType* sResource); - alignas(Get_CacheLineSize()) std::size_t dDebugLevel = 0; + alignas(JANA2_CACHE_LINE_BYTES) std::size_t dDebugLevel = 0; //static class members have external linkage: same instance shared between every translation unit (would be globally, put only private access) - alignas(Get_CacheLineSize()) static std::atomic dPoolLock; + alignas(JANA2_CACHE_LINE_BYTES) static std::atomic dPoolLock; alignas(Get_CacheLineSize()) static std::vector mResourcePool; alignas(Get_CacheLineSize()) static std::size_t dMaxPoolSize; alignas(Get_CacheLineSize()) static std::size_t dPoolCounter; //must be accessed within a lock due to how it's used in destructor: freeing all resources @@ -148,11 +148,11 @@ template class JSharedPtrRecycler //STATIC MEMBER DEFINITIONS //Since these are part of a template, these statics will only be defined once, no matter how much this header is included -template std::atomic JResourcePool::dPoolLock{0}; -template std::vector JResourcePool::mResourcePool = {}; -template std::size_t JResourcePool::dMaxPoolSize{10}; -template std::size_t JResourcePool::dPoolCounter{0}; -template std::atomic JResourcePool::dObjectCounter{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::atomic JResourcePool::dPoolLock{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::vector JResourcePool::mResourcePool = {}; +template alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool::dMaxPoolSize{10}; +template alignas(JANA2_CACHE_LINE_BYTES) std::size_t JResourcePool::dPoolCounter{0}; +template alignas(JANA2_CACHE_LINE_BYTES) std::atomic JResourcePool::dObjectCounter{0}; //CONSTRUCTORS template JResourcePool::JResourcePool(std::size_t sMaxPoolSize, std::size_t sDebugLevel) : JResourcePool() From 71ae29b1011a83feb842e350eb7936889d85b273 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 12 Oct 2023 11:11:35 -0400 Subject: [PATCH 5/7] Excise JResourcePool::Get_CacheLineSize --- src/libraries/JANA/Utils/JCpuInfo.h | 9 ++++++++- src/libraries/JANA/Utils/JResourcePool.h | 12 ------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/libraries/JANA/Utils/JCpuInfo.h b/src/libraries/JANA/Utils/JCpuInfo.h index 83b5d504b..004f0067f 100644 --- a/src/libraries/JANA/Utils/JCpuInfo.h +++ b/src/libraries/JANA/Utils/JCpuInfo.h @@ -6,6 +6,13 @@ #ifndef JANA2_CACHE_LINE_BYTES #define JANA2_CACHE_LINE_BYTES 64 #endif +// The cache line size is 64 for ifarm1402. gcc won't allow larger than 128 +// You can find the cache line size in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size +/*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on. + This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line. + This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well. + See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */ + #include @@ -23,4 +30,4 @@ namespace JCpuInfo { bool PinThreadToCpu(std::thread *thread, size_t cpu_id); -} \ No newline at end of file +} diff --git a/src/libraries/JANA/Utils/JResourcePool.h b/src/libraries/JANA/Utils/JResourcePool.h index d9df91bc3..06d8a8e1a 100644 --- a/src/libraries/JANA/Utils/JResourcePool.h +++ b/src/libraries/JANA/Utils/JResourcePool.h @@ -89,18 +89,6 @@ template class JResourcePool std::size_t Get_PoolSize(void) const; std::size_t Get_NumObjectsAllThreads(void) const{return dObjectCounter;} - static constexpr unsigned int Get_CacheLineSize(void) - { - /// Returns the cache line size for the processor of the target platform. - /*! The cache line size is useful for creating a buffer to make sure that a variable accessed by multiple threads does not share the cache line it is on. - This is useful for variables that may be written-to by one of the threads, because the thread will acquire locked access to the entire cache line. - This blocks other threads from operating on the other data stored on the cache line. Note that it is also important to align the shared data as well. - See http://www.drdobbs.com/parallel/eliminate-false-sharing/217500206?pgno=4 for more details. */ - - //cache line size is 64 for ifarm1402, gcc won't allow larger than 128 - //the cache line size is in /sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size - return JANA2_CACHE_LINE_BYTES ; //units are in bytes - } private: From f02e910a26609653ef6878ac809423af0656ea33 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 12 Oct 2023 11:23:20 -0400 Subject: [PATCH 6/7] Remove JApplication dependency on JResourcePool This hasn't been needed for a long time, it increases compilation times, and it has ended up being the source of many weird errors relating to alignas(). --- src/libraries/JANA/JApplication.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/JANA/JApplication.h b/src/libraries/JANA/JApplication.h index 1e37a2467..8cdd61c80 100644 --- a/src/libraries/JANA/JApplication.h +++ b/src/libraries/JANA/JApplication.h @@ -25,7 +25,6 @@ extern JApplication* japp; #include #include #include -#include #include From c5ac416999cdd317fb765520eba3d6091ce311d7 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 12 Oct 2023 11:41:37 -0400 Subject: [PATCH 7/7] Fix missing header on Ubuntu --- src/libraries/JANA/JApplication.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/JANA/JApplication.h b/src/libraries/JANA/JApplication.h index 8cdd61c80..ccdc3e016 100644 --- a/src/libraries/JANA/JApplication.h +++ b/src/libraries/JANA/JApplication.h @@ -8,6 +8,7 @@ #include #include #include +#include class JApplication; class JEventProcessor; @@ -29,7 +30,7 @@ extern JApplication* japp; ////////////////////////////////////////////////////////////////////////////////////////////////// -/// JANA application class (singleton). +/// JANA application class /// /// The JApplication class serves as a central access point for getting to most things /// in the JANA application. It owns the JThreadManager, JParameterManager, etc.