From 8db151b02e8ed3a9892f11e45520d9aa420fe2f8 Mon Sep 17 00:00:00 2001 From: Konstantinos Parasyris Date: Tue, 27 Jun 2023 10:04:06 -0700 Subject: [PATCH 01/18] Add repeat (-rp) CLI parameter to artifially increase physics execution time --- examples/app/eos_idealgas.hpp | 20 +++++++++++--------- examples/main.cpp | 5 ++++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/app/eos_idealgas.hpp b/examples/app/eos_idealgas.hpp index d1af5e43..bc45c422 100644 --- a/examples/app/eos_idealgas.hpp +++ b/examples/app/eos_idealgas.hpp @@ -18,10 +18,11 @@ class IdealGas : public EOS { const FPType gamma_; const FPType specific_heat_; + const int repeat; public: - IdealGas(FPType gamma, FPType specific_heat) - : gamma_(gamma), specific_heat_(specific_heat) + IdealGas(FPType gamma, FPType specific_heat, int repeat) + : gamma_(gamma), specific_heat_(specific_heat), repeat(repeat) { } @@ -39,14 +40,15 @@ class IdealGas : public EOS { const FPType gamma = gamma_; const FPType specific_heat = specific_heat_; - - using mfem::ForallWrap; - MFEM_FORALL(i, length, { - pressure[i] = (gamma - 1) * density[i] * energy[i]; - soundspeed2[i] = gamma * (gamma - 1) * energy[i]; - bulkmod[i] = gamma * pressure[i]; - temperature[i] = energy[i] / specific_heat; + for ( int r = 0; r < repeat; r++){ + using mfem::ForallWrap; + MFEM_FORALL(i, length, { + pressure[i] = (gamma - 1) * density[i] * energy[i]; + soundspeed2[i] = gamma * (gamma - 1) * energy[i]; + bulkmod[i] = gamma * pressure[i]; + temperature[i] = energy[i] / specific_heat; }); + } } #ifdef __ENABLE_PERFFLOWASPECT__ diff --git a/examples/main.cpp b/examples/main.cpp index 99f4d3e5..60c13f78 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -347,7 +347,7 @@ int run(const char *device_name, for (int mat_idx = 0; mat_idx < num_mats; ++mat_idx) { EOS *base; if (eos_name == std::string("ideal_gas")) { - base = new IdealGas(1.6, 1.4); + base = new IdealGas(1.6, 1.4, repeats); } else if (eos_name == std::string("constant_host")) { base = new ConstantEOSOnHost(physics_host_alloc.c_str(), 1.0); } else { @@ -634,6 +634,7 @@ int main(int argc, char **argv) double avg = 0.5; double stdDev = 0.2; bool reqDB = false; + int repeats = 1; const char *pool = "default"; #ifdef __ENABLE_DB__ @@ -752,6 +753,8 @@ int main(int argc, char **argv) args.AddOption( &verbose, "-v", "--verbose", "-qu", "--quiet", "Print extra stuff"); + args.AddOption(&repeats, "-rp", "--repeats", "Number of repeats to perform on physics eval"); + args.AddOption(&pool, "-ptype", "--pool-type", From 8647945100234bd5c5698f9b8fc72dc1ebe5ef06 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 10:52:44 -0800 Subject: [PATCH 02/18] Add repeat (-rp) CLI parameter to artifially increase physics execution time --- src/AMSlib/wf/debug.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AMSlib/wf/debug.h b/src/AMSlib/wf/debug.h index 52d04739..c025e072 100644 --- a/src/AMSlib/wf/debug.h +++ b/src/AMSlib/wf/debug.h @@ -24,8 +24,8 @@ enum AMSVerbosity { AMSFATAL = 0x001, - AMSWARNING = 0x002, - AMSINFO = 0x004, + AMSINFO = 0x002, + AMSWARNING = 0x004, AMSDEBUG = 0x008, }; From 7c271e56fd067d24bfc74ba4f09b2498196b1dbc Mon Sep 17 00:00:00 2001 From: Konstantinos Parasyris Date: Thu, 7 Sep 2023 18:17:27 -0700 Subject: [PATCH 03/18] Version that allows inference to take place on a different device than physics --- examples/main.cpp | 20 +++-- src/AMSlib/AMS.cpp | 8 +- src/AMSlib/include/AMS.h | 3 +- src/AMSlib/wf/resource_manager.hpp | 2 +- src/AMSlib/wf/workflow.hpp | 124 ++++++++++++++++++++++------- 5 files changed, 118 insertions(+), 39 deletions(-) diff --git a/examples/main.cpp b/examples/main.cpp index 60c13f78..13667ce1 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -175,7 +175,9 @@ int run(const char *device_name, CALIPER(mgr.start();) CALIPER(CALI_MARK_BEGIN("Setup");) - const bool use_device = std::strcmp(device_name, "cpu") != 0; + const bool physics_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); + const bool ml_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); + AMSDBType dbType = AMSDBType::None; if (std::strcmp(db_type, "csv") == 0) { dbType = AMSDBType::CSV; @@ -250,7 +252,7 @@ int run(const char *device_name, mfem::MemoryManager::SetUmpireHostAllocatorName(physics_host_alloc.c_str()); - if (use_device) { + if (physics_use_device) { mfem::MemoryManager::SetUmpireDeviceAllocatorName( physics_device_alloc.c_str()); } @@ -261,7 +263,7 @@ int run(const char *device_name, if (strcmp(pool, "default") != 0) { AMSSetAllocator(AMSResourceType::HOST, ams_host_alloc.c_str()); - if (use_device) { + if ( physics_use_device || ml_use_device ) { AMSSetAllocator(AMSResourceType::DEVICE, ams_device_alloc.c_str()); AMSSetAllocator(AMSResourceType::PINNED, ams_pinned_alloc.c_str()); } @@ -332,8 +334,12 @@ int run(const char *device_name, db_path = (strlen(db_config) > 0) ? db_config : nullptr; - AMSResourceType ams_device = AMSResourceType::HOST; - if (use_device) ams_device = AMSResourceType::DEVICE; + AMSResourceType ams_physics_device = AMSResourceType::HOST; + if (physics_use_device) ams_physics_device = AMSResourceType::DEVICE; + + AMSResourceType ams_ml_device = AMSResourceType::HOST; + if (ml_use_device) ams_ml_device = AMSResourceType::DEVICE; + AMSExecPolicy ams_loadBalance = AMSExecPolicy::UBALANCED; if (lbalance) ams_loadBalance = AMSExecPolicy::BALANCED; #else @@ -605,7 +611,7 @@ int main(int argc, char **argv) std::cout.setstate(std::ios::failbit); } - const char *device_name = "cpu"; + const char *device_name = "cpu_cpu"; const char *eos_name = "ideal_gas"; const char *model_path = ""; const char *hdcache_path = ""; @@ -647,7 +653,7 @@ int main(int argc, char **argv) // setup command line parser // ------------------------------------------------------------------------- mfem::OptionsParser args(argc, argv); - args.AddOption(&device_name, "-d", "--device", "Device config string"); + args.AddOption(&device_name, "-d", "--device", "device(physics)_device(model), for example: cpu_gpu means execute physics on the cpu and the model on gpu"); // set precision args.AddOption(&precision_opt, diff --git a/src/AMSlib/AMS.cpp b/src/AMSlib/AMS.cpp index 8b8308cf..33650765 100644 --- a/src/AMSlib/AMS.cpp +++ b/src/AMSlib/AMS.cpp @@ -87,7 +87,12 @@ AMSExecutor AMSCreateExecutor(const AMSConfig config) config.SPath, config.DBPath, config.dbType, +<<<<<<< HEAD:src/AMSlib/AMS.cpp config.device, +======= + config.physics_device, + config.ml_device, +>>>>>>> 85c492f (Version that allows inference to take place on a different device than physics):src/AMS.cpp config.threshold, config.uqPolicy, config.nClusters, @@ -105,7 +110,8 @@ AMSExecutor AMSCreateExecutor(const AMSConfig config) config.SPath, config.DBPath, config.dbType, - config.device, + config.physics_device, + config.ml_device, static_cast(config.threshold), config.uqPolicy, config.nClusters, diff --git a/src/AMSlib/include/AMS.h b/src/AMSlib/include/AMS.h index 64488948..f1168e51 100644 --- a/src/AMSlib/include/AMS.h +++ b/src/AMSlib/include/AMS.h @@ -71,7 +71,8 @@ enum struct AMSUQPolicy { typedef struct ams_conf { const AMSExecPolicy ePolicy; const AMSDType dType; - const AMSResourceType device; + const AMSResourceType physics_device; + const AMSResourceType ml_device; const AMSDBType dbType; AMSPhysicFn cBack; char *SPath; diff --git a/src/AMSlib/wf/resource_manager.hpp b/src/AMSlib/wf/resource_manager.hpp index a26b7d9f..572c5fe7 100644 --- a/src/AMSlib/wf/resource_manager.hpp +++ b/src/AMSlib/wf/resource_manager.hpp @@ -145,7 +145,7 @@ class ResourceManager static void deallocate(std::vector& dPtr, AMSResourceType resource) { for (auto* I : dPtr) - RMAllocators[resource]->deallocate(I); + RMAllocators[resource]->deallocate(I, dev); } static void init() diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index a20b4aaa..4fe7cb36 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -68,8 +68,9 @@ class AMSWorkflow * (world_size for MPI) */ int wSize; - /** @brief Location of the original application data (CPU or GPU) */ - AMSResourceType appDataLoc; + /** @brief Location of data during simulation (CPU or GPU) */ + AMSResourceType mlLoc; + AMSResourceType phLoc; /** @brief execution policy of the distributed system. Load balance or not. */ const AMSExecPolicy ePolicy; @@ -101,8 +102,7 @@ class AMSWorkflow std::vector hInputs, hOutputs; - if (appDataLoc == AMSResourceType::HOST) - return DB->store(num_elements, inputs, outputs); + if ( phLoc == AMSResourceType::HOST ) return DB->store(num_elements, inputs, outputs); // Compute number of elements that fit inside the buffer size_t bElements = bSize / sizeof(FPTypeValue); @@ -150,9 +150,10 @@ class AMSWorkflow : AppCall(nullptr), DB(nullptr), dbType(AMSDBType::None), - appDataLoc(AMSResourceType::HOST), ePolicy(AMSExecPolicy::UBALANCED) { + phLoc = AMSResourceType::HOST; + mlLoc = AMSResourceType::DEVICE; #ifdef __ENABLE_DB__ DB = createDB("miniApp_data.txt", dbType, 0); @@ -165,7 +166,8 @@ class AMSWorkflow char *surrogate_path, char *db_path, const AMSDBType dbType, - AMSResourceType appDataLoc, + const AMSResourceType physics_device, + const AMSResourceType ml_device, FPTypeValue threshold, const AMSUQPolicy uqPolicy, const int nClusters, @@ -176,7 +178,8 @@ class AMSWorkflow dbType(dbType), rId(_pId), wSize(_wSize), - appDataLoc(appDataLoc), + phLoc(physics_device), + mlLoc(ml_device), uqPolicy(uqPolicy), ePolicy(policy) { @@ -195,6 +198,70 @@ class AMSWorkflow ~AMSWorkflow() { DBG(Workflow, "Destroying Workflow Handler"); } + void ml_step(std::vector& Inputs, + std::vector& Outputs, + int totalElements, + bool *_p_ml_acceptable) { + // The predicate with which we will split the data on a later step + + bool *p_ml_acceptable = _p_ml_acceptable; + std::vector tmpInputs = Inputs; + std::vector tmpOutputs = Outputs; + if ( phLoc != mlLoc ){ + p_ml_acceptable = ams::ResourceManager::allocate(totalElements, mlLoc); + + for (int i = 0; i < Inputs.size(); i++) + tmpInputs[i] = + ams::ResourceManager::allocate(totalElements, mlLoc); + for (int i = 0; i < Outputs.size(); i++) + tmpOutputs[i] = + ams::ResourceManager::allocate(totalElements, mlLoc); + + for (int i = 0; i < Inputs.size(); i++) { + ResourceManager::copy( const_cast (Inputs[i]), + const_cast(tmpInputs[i])); + } + } + + // ------------------------------------------------------------- + // STEP 1: call the hdcache to look at input uncertainties + // to decide if making a ML inference makes sense + // ------------------------------------------------------------- + if (hdcache != nullptr) { + CALIPER(CALI_MARK_BEGIN("UQ_MODULE");) + hdcache->evaluate(totalElements, tmpInputs, p_ml_acceptable); + CALIPER(CALI_MARK_END("UQ_MODULE");) + } + + DBG(Workflow, "Computed Predicates") + + CALIPER(CALI_MARK_BEGIN("SURROGATE");) + // We need to call the model on all data values. + // Because we expect it to be faster. + // I guess we may need to add some policy to do this + DBG(Workflow, "Model exists, I am calling surrogate (for all data)"); + surrogate->evaluate(totalElements, tmpInputs, tmpOutputs); + CALIPER(CALI_MARK_END("SURROGATE");) + + if ( phLoc != mlLoc ){ + // Copy out the result + ResourceManager::copy(p_ml_acceptable, _p_ml_acceptable); + ResourceManager::deallocate(p_ml_acceptable, mlLoc); + for (int i = 0; i < Outputs.size(); i++){ + ResourceManager::copy(tmpOutputs[i], Outputs[i]); + + // De-allocate temp data + for (int i = 0; i < Inputs.size(); i++) + ams::ResourceManager::deallocate(const_cast(tmpInputs[i]), mlLoc); + for (int i = 0; i < Outputs.size(); i++) + ams::ResourceManager::deallocate((tmpOutputs[i]), mlLoc); + ams::ResourceManager::deallocate(p_ml_acceptable, mlLoc); + + } + } + + } + /** @brief This is the main entry point of AMSLib and replaces the original * execution path of the application. * @param[in] probDescr an opaque type that will be forwarded to the @@ -291,7 +358,9 @@ class AMSWorkflow UQModel->evaluate(totalElements, origInputs, origOutputs, p_ml_acceptable); CALIPER(CALI_MARK_END("UQ_MODULE");) - DBG(Workflow, "Computed Predicates") + + bool *p_ml_acceptable = ams::ResourceManager::allocate(totalElements, phLoc); + ml_step(origInputs, origOutputs, totalElements, p_ml_acceptable); // Pointer values which store input data values // to be computed using the eos function. @@ -303,8 +372,6 @@ class AMSWorkflow appDataLoc)); } - DBG(Workflow, "Allocated input resources") - bool *predicate = p_ml_acceptable; // ----------------------------------------------------------------- @@ -328,18 +395,17 @@ class AMSWorkflow void **oPtr = reinterpret_cast(packedOutputs.data()); long lbElements = packedElements; -#ifdef __ENABLE_MPI__ - CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) - AMSLoadBalancer lBalancer( - rId, wSize, packedElements, Comm, inputDim, outputDim, appDataLoc); - if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.scatterInputs(packedInputs, appDataLoc); - iPtr = reinterpret_cast(lBalancer.inputs()); - oPtr = reinterpret_cast(lBalancer.outputs()); - lbElements = lBalancer.getBalancedSize(); - } - CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) -#endif +//#ifdef __ENABLE_MPI__ +// CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) +// AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mLoc); +// if (ePolicy == AMSExecPolicy::BALANCED && Comm) { +// lBalancer.scatterInputs(packedInputs, mLoc); +// iPtr = reinterpret_cast(lBalancer.inputs()); +// oPtr = reinterpret_cast(lBalancer.outputs()); +// lbElements = lBalancer.getBalancedSize(); +// } +// CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) +//#endif // ---- 3b: call the physics module and store in the data base if (packedElements > 0) { @@ -348,13 +414,13 @@ class AMSWorkflow CALIPER(CALI_MARK_END("PHYSICS MODULE");) } -#ifdef __ENABLE_MPI__ - CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) - if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.gatherOutputs(packedOutputs, appDataLoc); - } - CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) -#endif +//#ifdef __ENABLE_MPI__ +// CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) +// if (ePolicy == AMSExecPolicy::BALANCED && Comm) { +// lBalancer.gatherOutputs(packedOutputs, mLoc); +// } +// CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) +//#endif } // ---- 3c: unpack the data From f193bc5bdfb00b2e851a2c2473020be1cae2035b Mon Sep 17 00:00:00 2001 From: Konstantinos Parasyris Date: Mon, 11 Sep 2023 08:04:17 -0700 Subject: [PATCH 04/18] Support cpu_cpu/cpu_gpu/gpu_cpu/gpu_gpu execution --- examples/main.cpp | 9 ++++++++- src/AMSlib/ml/surrogate.hpp | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/main.cpp b/examples/main.cpp index 13667ce1..aeea7bd2 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -178,6 +178,7 @@ int run(const char *device_name, const bool physics_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); const bool ml_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); + AMSDBType dbType = AMSDBType::None; if (std::strcmp(db_type, "csv") == 0) { dbType = AMSDBType::CSV; @@ -272,7 +273,13 @@ int run(const char *device_name, mfem::Device::SetMemoryTypes(mfem::MemoryType::HOST_UMPIRE, mfem::MemoryType::DEVICE_UMPIRE); - mfem::Device device(device_name); + mfem::Device device; + + if ( physics_use_device ) + device.Configure("cuda"); + else + device.Configure("cpu"); + std::cout << std::endl; device.Print(); std::cout << std::endl; diff --git a/src/AMSlib/ml/surrogate.hpp b/src/AMSlib/ml/surrogate.hpp index 1514dd4f..c83e06e4 100644 --- a/src/AMSlib/ml/surrogate.hpp +++ b/src/AMSlib/ml/surrogate.hpp @@ -156,7 +156,7 @@ class SurrogateModel inline void _load(const std::string& model_path, const std::string& device_name) { - DBG(Surrogate, "Using model at double precision"); + DBG(Surrogate, "Using model(%s) at double precision at device %s", model_path.c_str(), device_name.c_str()); _load_torch(model_path, torch::Device(device_name), torch::kFloat64); } @@ -166,7 +166,7 @@ class SurrogateModel inline void _load(const std::string& model_path, const std::string& device_name) { - DBG(Surrogate, "Using model at single precision"); + DBG(Surrogate, "Using model (%s) at single precision at device %s", model_path.c_str(), device_name.c_str()) _load_torch(model_path, torch::Device(device_name), torch::kFloat32); } From b6eb2d2869bec2a3776d92e21fd0643214d3cbfc Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 15:55:35 -0800 Subject: [PATCH 05/18] uncomment MPI functionality --- src/AMSlib/wf/workflow.hpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 4fe7cb36..82b926b2 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -395,17 +395,17 @@ class AMSWorkflow void **oPtr = reinterpret_cast(packedOutputs.data()); long lbElements = packedElements; -//#ifdef __ENABLE_MPI__ -// CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) -// AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mLoc); -// if (ePolicy == AMSExecPolicy::BALANCED && Comm) { -// lBalancer.scatterInputs(packedInputs, mLoc); -// iPtr = reinterpret_cast(lBalancer.inputs()); -// oPtr = reinterpret_cast(lBalancer.outputs()); -// lbElements = lBalancer.getBalancedSize(); -// } -// CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) -//#endif +#ifdef __ENABLE_MPI__ + CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) + AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mLoc); + if (ePolicy == AMSExecPolicy::BALANCED && Comm) { + lBalancer.scatterInputs(packedInputs, mLoc); + iPtr = reinterpret_cast(lBalancer.inputs()); + oPtr = reinterpret_cast(lBalancer.outputs()); + lbElements = lBalancer.getBalancedSize(); + } + CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) +#endif // ---- 3b: call the physics module and store in the data base if (packedElements > 0) { @@ -414,13 +414,13 @@ class AMSWorkflow CALIPER(CALI_MARK_END("PHYSICS MODULE");) } -//#ifdef __ENABLE_MPI__ -// CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) -// if (ePolicy == AMSExecPolicy::BALANCED && Comm) { -// lBalancer.gatherOutputs(packedOutputs, mLoc); -// } -// CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) -//#endif +#ifdef __ENABLE_MPI__ + CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) + if (ePolicy == AMSExecPolicy::BALANCED && Comm) { + lBalancer.gatherOutputs(packedOutputs, mLoc); + } + CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) +#endif } // ---- 3c: unpack the data From 6f639a85f03315ee876f13b5d4f9dfdc7c668348 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 16:01:34 -0800 Subject: [PATCH 06/18] resolve conflict (pass physics and ML params --- src/AMSlib/AMS.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/AMSlib/AMS.cpp b/src/AMSlib/AMS.cpp index 33650765..97bdcab5 100644 --- a/src/AMSlib/AMS.cpp +++ b/src/AMSlib/AMS.cpp @@ -87,12 +87,8 @@ AMSExecutor AMSCreateExecutor(const AMSConfig config) config.SPath, config.DBPath, config.dbType, -<<<<<<< HEAD:src/AMSlib/AMS.cpp - config.device, -======= config.physics_device, config.ml_device, ->>>>>>> 85c492f (Version that allows inference to take place on a different device than physics):src/AMS.cpp config.threshold, config.uqPolicy, config.nClusters, From 58ffd788c1f0fb900a61328aff2051b899f610ea Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 21:37:30 -0800 Subject: [PATCH 07/18] reverting change --- src/AMSlib/wf/debug.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AMSlib/wf/debug.h b/src/AMSlib/wf/debug.h index c025e072..52d04739 100644 --- a/src/AMSlib/wf/debug.h +++ b/src/AMSlib/wf/debug.h @@ -24,8 +24,8 @@ enum AMSVerbosity { AMSFATAL = 0x001, - AMSINFO = 0x002, - AMSWARNING = 0x004, + AMSWARNING = 0x002, + AMSINFO = 0x004, AMSDEBUG = 0x008, }; From 8f8b84a60a24a7702532fbd16080f95a0323f5b4 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 21:57:24 -0800 Subject: [PATCH 08/18] Update the brief for params --- src/AMSlib/wf/workflow.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 82b926b2..e27ae096 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -68,7 +68,7 @@ class AMSWorkflow * (world_size for MPI) */ int wSize; - /** @brief Location of data during simulation (CPU or GPU) */ + /** @brief mlLoc describes the location of the ML model during simulation (CPU or GPU) */ AMSResourceType mlLoc; AMSResourceType phLoc; From d7a069931d6d2a8007e711b79d6a3dde121db944 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 22:38:32 -0800 Subject: [PATCH 09/18] replace calls to hdcache and model with UQModel single call --- src/AMSlib/wf/workflow.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index e27ae096..1c6acea9 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -224,14 +224,12 @@ class AMSWorkflow } // ------------------------------------------------------------- - // STEP 1: call the hdcache to look at input uncertainties + // STEP 1: call the UQ module to look at input uncertainties // to decide if making a ML inference makes sense // ------------------------------------------------------------- - if (hdcache != nullptr) { - CALIPER(CALI_MARK_BEGIN("UQ_MODULE");) - hdcache->evaluate(totalElements, tmpInputs, p_ml_acceptable); - CALIPER(CALI_MARK_END("UQ_MODULE");) - } + CALIPER(CALI_MARK_BEGIN("UQ_MODULE");) + UQModel->evaluate(totalElements, tmpInputs, tmpOutputs, p_ml_acceptable); + CALIPER(CALI_MARK_END("UQ_MODULE");) DBG(Workflow, "Computed Predicates") @@ -240,7 +238,7 @@ class AMSWorkflow // Because we expect it to be faster. // I guess we may need to add some policy to do this DBG(Workflow, "Model exists, I am calling surrogate (for all data)"); - surrogate->evaluate(totalElements, tmpInputs, tmpOutputs); + UQModel->evaluate(totalElements, tmpInputs, tmpOutputs); CALIPER(CALI_MARK_END("SURROGATE");) if ( phLoc != mlLoc ){ From 5beca79942ac560ee4e1b01ae610940d0ff29e7d Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 29 Jan 2024 22:41:15 -0800 Subject: [PATCH 10/18] remove param 'dev' --- src/AMSlib/wf/resource_manager.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AMSlib/wf/resource_manager.hpp b/src/AMSlib/wf/resource_manager.hpp index 572c5fe7..a26b7d9f 100644 --- a/src/AMSlib/wf/resource_manager.hpp +++ b/src/AMSlib/wf/resource_manager.hpp @@ -145,7 +145,7 @@ class ResourceManager static void deallocate(std::vector& dPtr, AMSResourceType resource) { for (auto* I : dPtr) - RMAllocators[resource]->deallocate(I, dev); + RMAllocators[resource]->deallocate(I); } static void init() From c776f33e60a7d98e42deea689a43a918b7f827d1 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Tue, 30 Jan 2024 00:55:35 -0800 Subject: [PATCH 11/18] Fixes: param names, number of params, etc. --- examples/main.cpp | 9 ++++++--- src/AMSlib/wf/workflow.hpp | 37 +++++++++++++------------------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/examples/main.cpp b/examples/main.cpp index aeea7bd2..4af7b6b8 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -166,7 +166,8 @@ int run(const char *device_name, const char *model_path, const char *db_config, bool lbalance, - int k_nearest) + int k_nearest, + int repeats) { // ------------------------------------------------------------------------- // setup @@ -860,7 +861,8 @@ int main(int argc, char **argv) model_path, db_config, lbalance, - k_nearest); + k_nearest, + repeats); else if (precision == AMSDType::Double) ret = run(device_name, db_type, @@ -886,7 +888,8 @@ int main(int argc, char **argv) model_path, db_config, lbalance, - k_nearest); + k_nearest, + repeats); else { std::cerr << "Invalid precision " << precision_opt << "\n"; return -1; diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 1c6acea9..84232775 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -190,7 +190,7 @@ class AMSWorkflow } UQModel = std::make_unique>( - appDataLoc, uqPolicy, uq_path, nClusters, surrogate_path, threshold); + phLoc, uqPolicy, uq_path, nClusters, surrogate_path, threshold); } void set_physics(AMSPhysicFn _AppCall) { AppCall = _AppCall; } @@ -238,7 +238,7 @@ class AMSWorkflow // Because we expect it to be faster. // I guess we may need to add some policy to do this DBG(Workflow, "Model exists, I am calling surrogate (for all data)"); - UQModel->evaluate(totalElements, tmpInputs, tmpOutputs); + UQModel->evaluate(totalElements, tmpInputs, tmpOutputs, p_ml_acceptable); CALIPER(CALI_MARK_END("SURROGATE");) if ( phLoc != mlLoc ){ @@ -346,18 +346,7 @@ class AMSWorkflow } // The predicate with which we will split the data on a later step bool *p_ml_acceptable = - ams::ResourceManager::allocate(totalElements, appDataLoc); - - // ------------------------------------------------------------- - // STEP 1: call the UQ module to look at input uncertainties - // to decide if making a ML inference makes sense - // ------------------------------------------------------------- - CALIPER(CALI_MARK_BEGIN("UQ_MODULE");) - UQModel->evaluate(totalElements, origInputs, origOutputs, p_ml_acceptable); - CALIPER(CALI_MARK_END("UQ_MODULE");) - - - bool *p_ml_acceptable = ams::ResourceManager::allocate(totalElements, phLoc); + ams::ResourceManager::allocate(totalElements, phLoc); ml_step(origInputs, origOutputs, totalElements, p_ml_acceptable); // Pointer values which store input data values @@ -367,7 +356,7 @@ class AMSWorkflow for (int i = 0; i < inputDim; i++) { packedInputs.emplace_back( ams::ResourceManager::allocate(totalElements, - appDataLoc)); + phLoc)); } bool *predicate = p_ml_acceptable; @@ -377,7 +366,7 @@ class AMSWorkflow // ----------------------------------------------------------------- // ---- 3a: we need to pack the sparse data based on the uq flag const long packedElements = data_handler::pack( - appDataLoc, predicate, totalElements, origInputs, packedInputs); + phLoc, predicate, totalElements, origInputs, packedInputs); // Pointer values which store output data values // to be computed using the eos function. @@ -385,7 +374,7 @@ class AMSWorkflow for (int i = 0; i < outputDim; i++) { packedOutputs.emplace_back( ams::ResourceManager::allocate(packedElements, - appDataLoc)); + phLoc)); } { @@ -395,9 +384,9 @@ class AMSWorkflow #ifdef __ENABLE_MPI__ CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) - AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mLoc); + AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mlLoc); if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.scatterInputs(packedInputs, mLoc); + lBalancer.scatterInputs(packedInputs, mlLoc); iPtr = reinterpret_cast(lBalancer.inputs()); oPtr = reinterpret_cast(lBalancer.outputs()); lbElements = lBalancer.getBalancedSize(); @@ -415,7 +404,7 @@ class AMSWorkflow #ifdef __ENABLE_MPI__ CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.gatherOutputs(packedOutputs, mLoc); + lBalancer.gatherOutputs(packedOutputs, mlLoc); } CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) #endif @@ -423,7 +412,7 @@ class AMSWorkflow // ---- 3c: unpack the data data_handler::unpack( - appDataLoc, predicate, totalElements, packedOutputs, origOutputs); + phLoc, predicate, totalElements, packedOutputs, origOutputs); DBG(Workflow, "Finished physics evaluation") @@ -440,11 +429,11 @@ class AMSWorkflow // Deallocate temporal data // ----------------------------------------------------------------- for (int i = 0; i < inputDim; i++) - ams::ResourceManager::deallocate(packedInputs[i], appDataLoc); + ams::ResourceManager::deallocate(packedInputs[i], phLoc); for (int i = 0; i < outputDim; i++) - ams::ResourceManager::deallocate(packedOutputs[i], appDataLoc); + ams::ResourceManager::deallocate(packedOutputs[i], phLoc); - ams::ResourceManager::deallocate(p_ml_acceptable, appDataLoc); + ams::ResourceManager::deallocate(p_ml_acceptable, phLoc); DBG(Workflow, "Finished AMSExecution") CINFO(Workflow, From fea8eca02dadc6b995e0760e9be135a5b1174b68 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Tue, 30 Jan 2024 09:00:00 -0800 Subject: [PATCH 12/18] fix order to cpu_gpu for device_name --- examples/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/main.cpp b/examples/main.cpp index 4af7b6b8..918d5a02 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -177,7 +177,7 @@ int run(const char *device_name, CALIPER(CALI_MARK_BEGIN("Setup");) const bool physics_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); - const bool ml_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); + const bool ml_use_device = ((std::strcmp(device_name, "cpu_gpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); AMSDBType dbType = AMSDBType::None; From dc8026382e2823e2af14951f1564afafbea6a176 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Tue, 30 Jan 2024 15:36:36 -0800 Subject: [PATCH 13/18] prameter fixes and formatting --- examples/main.cpp | 21 +++++++--- src/AMSlib/wf/workflow.hpp | 83 ++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/examples/main.cpp b/examples/main.cpp index 918d5a02..683e300b 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -176,8 +176,10 @@ int run(const char *device_name, CALIPER(mgr.start();) CALIPER(CALI_MARK_BEGIN("Setup");) - const bool physics_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); - const bool ml_use_device = ((std::strcmp(device_name, "cpu_gpu") == 0) || std::strcmp(device_name, "gpu_gpu") == 0); + const bool physics_use_device = ((std::strcmp(device_name, "gpu_cpu") == 0) || + std::strcmp(device_name, "gpu_gpu") == 0); + const bool ml_use_device = ((std::strcmp(device_name, "cpu_gpu") == 0) || + std::strcmp(device_name, "gpu_gpu") == 0); AMSDBType dbType = AMSDBType::None; @@ -265,7 +267,7 @@ int run(const char *device_name, if (strcmp(pool, "default") != 0) { AMSSetAllocator(AMSResourceType::HOST, ams_host_alloc.c_str()); - if ( physics_use_device || ml_use_device ) { + if (physics_use_device || ml_use_device) { AMSSetAllocator(AMSResourceType::DEVICE, ams_device_alloc.c_str()); AMSSetAllocator(AMSResourceType::PINNED, ams_pinned_alloc.c_str()); } @@ -276,7 +278,7 @@ int run(const char *device_name, mfem::Device device; - if ( physics_use_device ) + if (physics_use_device) device.Configure("cuda"); else device.Configure("cpu"); @@ -661,7 +663,11 @@ int main(int argc, char **argv) // setup command line parser // ------------------------------------------------------------------------- mfem::OptionsParser args(argc, argv); - args.AddOption(&device_name, "-d", "--device", "device(physics)_device(model), for example: cpu_gpu means execute physics on the cpu and the model on gpu"); + args.AddOption(&device_name, + "-d", + "--device", + "device(physics)_device(model), for example: cpu_gpu means " + "execute physics on the cpu and the model on gpu"); // set precision args.AddOption(&precision_opt, @@ -767,7 +773,10 @@ int main(int argc, char **argv) args.AddOption( &verbose, "-v", "--verbose", "-qu", "--quiet", "Print extra stuff"); - args.AddOption(&repeats, "-rp", "--repeats", "Number of repeats to perform on physics eval"); + args.AddOption(&repeats, + "-rp", + "--repeats", + "Number of repeats to perform on physics eval"); args.AddOption(&pool, "-ptype", diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 84232775..587fedab 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -70,6 +70,7 @@ class AMSWorkflow /** @brief mlLoc describes the location of the ML model during simulation (CPU or GPU) */ AMSResourceType mlLoc; + /** @brief phLoc describes the location of the phycics module during simulation (CPU or GPU) */ AMSResourceType phLoc; /** @brief execution policy of the distributed system. Load balance or not. */ @@ -102,7 +103,8 @@ class AMSWorkflow std::vector hInputs, hOutputs; - if ( phLoc == AMSResourceType::HOST ) return DB->store(num_elements, inputs, outputs); + if (phLoc == AMSResourceType::HOST) + return DB->store(num_elements, inputs, outputs); // Compute number of elements that fit inside the buffer size_t bElements = bSize / sizeof(FPTypeValue); @@ -152,8 +154,8 @@ class AMSWorkflow dbType(AMSDBType::None), ePolicy(AMSExecPolicy::UBALANCED) { - phLoc = AMSResourceType::HOST; - mlLoc = AMSResourceType::DEVICE; + phLoc = AMSResourceType::HOST; + mlLoc = AMSResourceType::DEVICE; #ifdef __ENABLE_DB__ DB = createDB("miniApp_data.txt", dbType, 0); @@ -190,7 +192,7 @@ class AMSWorkflow } UQModel = std::make_unique>( - phLoc, uqPolicy, uq_path, nClusters, surrogate_path, threshold); + mlLoc, uqPolicy, uq_path, nClusters, surrogate_path, threshold); } void set_physics(AMSPhysicFn _AppCall) { AppCall = _AppCall; } @@ -198,28 +200,30 @@ class AMSWorkflow ~AMSWorkflow() { DBG(Workflow, "Destroying Workflow Handler"); } - void ml_step(std::vector& Inputs, - std::vector& Outputs, + void ml_step(std::vector &Inputs, + std::vector &Outputs, int totalElements, - bool *_p_ml_acceptable) { + bool *_p_ml_acceptable) + { // The predicate with which we will split the data on a later step bool *p_ml_acceptable = _p_ml_acceptable; std::vector tmpInputs = Inputs; std::vector tmpOutputs = Outputs; - if ( phLoc != mlLoc ){ - p_ml_acceptable = ams::ResourceManager::allocate(totalElements, mlLoc); + if (phLoc != mlLoc) { + p_ml_acceptable = + ams::ResourceManager::allocate(totalElements, mlLoc); for (int i = 0; i < Inputs.size(); i++) tmpInputs[i] = - ams::ResourceManager::allocate(totalElements, mlLoc); + ams::ResourceManager::allocate(totalElements, mlLoc); for (int i = 0; i < Outputs.size(); i++) tmpOutputs[i] = - ams::ResourceManager::allocate(totalElements, mlLoc); + ams::ResourceManager::allocate(totalElements, mlLoc); for (int i = 0; i < Inputs.size(); i++) { - ResourceManager::copy( const_cast (Inputs[i]), - const_cast(tmpInputs[i])); + ResourceManager::copy(const_cast(Inputs[i]), + const_cast(tmpInputs[i])); } } @@ -241,23 +245,23 @@ class AMSWorkflow UQModel->evaluate(totalElements, tmpInputs, tmpOutputs, p_ml_acceptable); CALIPER(CALI_MARK_END("SURROGATE");) - if ( phLoc != mlLoc ){ + if (phLoc != mlLoc) { // Copy out the result ResourceManager::copy(p_ml_acceptable, _p_ml_acceptable); ResourceManager::deallocate(p_ml_acceptable, mlLoc); - for (int i = 0; i < Outputs.size(); i++){ + for (int i = 0; i < Outputs.size(); i++) { ResourceManager::copy(tmpOutputs[i], Outputs[i]); - // De-allocate temp data - for (int i = 0; i < Inputs.size(); i++) - ams::ResourceManager::deallocate(const_cast(tmpInputs[i]), mlLoc); - for (int i = 0; i < Outputs.size(); i++) + // De-allocate temp data + for (int i = 0; i < Inputs.size(); i++) + ams::ResourceManager::deallocate(const_cast( + tmpInputs[i]), + mlLoc); + for (int i = 0; i < Outputs.size(); i++) ams::ResourceManager::deallocate((tmpOutputs[i]), mlLoc); - ams::ResourceManager::deallocate(p_ml_acceptable, mlLoc); - + ams::ResourceManager::deallocate(p_ml_acceptable, mlLoc); } } - } /** @brief This is the main entry point of AMSLib and replaces the original @@ -355,8 +359,7 @@ class AMSWorkflow for (int i = 0; i < inputDim; i++) { packedInputs.emplace_back( - ams::ResourceManager::allocate(totalElements, - phLoc)); + ams::ResourceManager::allocate(totalElements, phLoc)); } bool *predicate = p_ml_acceptable; @@ -373,8 +376,7 @@ class AMSWorkflow std::vector packedOutputs; for (int i = 0; i < outputDim; i++) { packedOutputs.emplace_back( - ams::ResourceManager::allocate(packedElements, - phLoc)); + ams::ResourceManager::allocate(packedElements, phLoc)); } { @@ -383,15 +385,16 @@ class AMSWorkflow long lbElements = packedElements; #ifdef __ENABLE_MPI__ - CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) - AMSLoadBalancer lBalancer(rId, wSize, packedElements, Comm, inputDim, outputDim, mlLoc); - if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.scatterInputs(packedInputs, mlLoc); - iPtr = reinterpret_cast(lBalancer.inputs()); - oPtr = reinterpret_cast(lBalancer.outputs()); - lbElements = lBalancer.getBalancedSize(); - } - CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) + CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) + AMSLoadBalancer lBalancer( + rId, wSize, packedElements, Comm, inputDim, outputDim, mlLoc); + if (ePolicy == AMSExecPolicy::BALANCED && Comm) { + lBalancer.scatterInputs(packedInputs, mlLoc); + iPtr = reinterpret_cast(lBalancer.inputs()); + oPtr = reinterpret_cast(lBalancer.outputs()); + lbElements = lBalancer.getBalancedSize(); + } + CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) #endif // ---- 3b: call the physics module and store in the data base @@ -402,11 +405,11 @@ class AMSWorkflow } #ifdef __ENABLE_MPI__ - CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) - if (ePolicy == AMSExecPolicy::BALANCED && Comm) { - lBalancer.gatherOutputs(packedOutputs, mlLoc); - } - CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) + CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) + if (ePolicy == AMSExecPolicy::BALANCED && Comm) { + lBalancer.gatherOutputs(packedOutputs, mlLoc); + } + CALIPER(CALI_MARK_END("LOAD BALANCE MODULE");) #endif } From 592eed97c66c1ac4f6fc315d0f4f0e9efb7569f4 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Tue, 30 Jan 2024 15:42:05 -0800 Subject: [PATCH 14/18] adding missing params --- examples/app/eos_ams.cpp | 1 + examples/app/eos_ams.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/app/eos_ams.cpp b/examples/app/eos_ams.cpp index 4e995a89..6dabfd5b 100644 --- a/examples/app/eos_ams.cpp +++ b/examples/app/eos_ams.cpp @@ -35,6 +35,7 @@ AMSEOS::AMSEOS(EOS *model, const int k_nearest, const int mpi_task, const int mpi_nproc, + const int repeats, const double threshold, const char *surrogate_path, const char *uq_path, diff --git a/examples/app/eos_ams.hpp b/examples/app/eos_ams.hpp index dbd8260c..1cbd3206 100644 --- a/examples/app/eos_ams.hpp +++ b/examples/app/eos_ams.hpp @@ -29,6 +29,7 @@ class AMSEOS : public EOS const int k_nearest, const int mpi_task, const int mpi_nproc, + const int repeats, const double threshold, const char *surrogate_path, const char *uq_path, From 68dc6808a6cf8efe446456c9d1321e134897c071 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Tue, 30 Jan 2024 22:45:22 -0800 Subject: [PATCH 15/18] remove param repeats --- examples/app/eos_ams.cpp | 1 - examples/app/eos_ams.hpp | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/app/eos_ams.cpp b/examples/app/eos_ams.cpp index 6dabfd5b..4e995a89 100644 --- a/examples/app/eos_ams.cpp +++ b/examples/app/eos_ams.cpp @@ -35,7 +35,6 @@ AMSEOS::AMSEOS(EOS *model, const int k_nearest, const int mpi_task, const int mpi_nproc, - const int repeats, const double threshold, const char *surrogate_path, const char *uq_path, diff --git a/examples/app/eos_ams.hpp b/examples/app/eos_ams.hpp index 1cbd3206..dbd8260c 100644 --- a/examples/app/eos_ams.hpp +++ b/examples/app/eos_ams.hpp @@ -29,7 +29,6 @@ class AMSEOS : public EOS const int k_nearest, const int mpi_task, const int mpi_nproc, - const int repeats, const double threshold, const char *surrogate_path, const char *uq_path, From 028c2f051396c942e54e539c40d780d12a36adb3 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Wed, 31 Jan 2024 11:31:03 -0800 Subject: [PATCH 16/18] pass params for both ML and physics device types --- examples/app/eos_ams.cpp | 6 ++++-- examples/app/eos_ams.hpp | 3 ++- examples/main.cpp | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/app/eos_ams.cpp b/examples/app/eos_ams.cpp index 4e995a89..c1a5e097 100644 --- a/examples/app/eos_ams.cpp +++ b/examples/app/eos_ams.cpp @@ -30,7 +30,8 @@ AMSEOS::AMSEOS(EOS *model, const AMSDBType db_type, const AMSDType dtype, const AMSExecPolicy exec_policy, - const AMSResourceType res_type, + const AMSResourceType res_type_physics, + const AMSResourceType res_type_ml, const AMSUQPolicy uq_policy, const int k_nearest, const int mpi_task, @@ -43,7 +44,8 @@ AMSEOS::AMSEOS(EOS *model, { AMSConfig conf = {exec_policy, dtype, - res_type, + res_type_physics, + res_type_ml, db_type, callBack, (char *)surrogate_path, diff --git a/examples/app/eos_ams.hpp b/examples/app/eos_ams.hpp index dbd8260c..541e413b 100644 --- a/examples/app/eos_ams.hpp +++ b/examples/app/eos_ams.hpp @@ -24,7 +24,8 @@ class AMSEOS : public EOS const AMSDBType db_type, const AMSDType dtype, const AMSExecPolicy exec_policy, - const AMSResourceType res_type, + const AMSResourceType res_type_physics, + const AMSResourceType res_type_ml, const AMSUQPolicy uq_policy, const int k_nearest, const int mpi_task, diff --git a/examples/main.cpp b/examples/main.cpp index 683e300b..9fe69481 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -376,7 +376,8 @@ int run(const char *device_name, dbType, precision, ams_loadBalance, - ams_device, + ams_physics_device, + ams_ml_device, uq_policy, k_nearest, rId, From f0c1b62825524b5d0fd37b09fa1078ec943051c4 Mon Sep 17 00:00:00 2001 From: Giorgis Georgakoudis Date: Mon, 5 Feb 2024 09:13:19 -0800 Subject: [PATCH 17/18] Fix calling the surrogate in random uq --- src/AMSlib/ml/uq.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/AMSlib/ml/uq.hpp b/src/AMSlib/ml/uq.hpp index 1935cb1e..dcd1e98a 100644 --- a/src/AMSlib/ml/uq.hpp +++ b/src/AMSlib/ml/uq.hpp @@ -128,6 +128,11 @@ class UQ DBG(Workflow, "Evaluating Random UQ"); randomUQ->evaluate(totalElements, p_ml_acceptable); CALIPER(CALI_MARK_END("RANDOM_UQ");) + + CALIPER(CALI_MARK_BEGIN("SURROGATE");) + DBG(Workflow, "Model exists, I am calling surrogate (for all data)"); + surrogate->evaluate(totalElements, inputs, outputs); + CALIPER(CALI_MARK_END("SURROGATE");) } else { THROW(std::runtime_error, "Invalid UQ policy"); } From 35b599f7c480cd5f0f6d5d753f64e19967bf1792 Mon Sep 17 00:00:00 2001 From: Tapasya Patki Date: Fri, 9 Feb 2024 19:19:54 -0800 Subject: [PATCH 18/18] Fixes for segfaults on cpu_gpu --- src/AMSlib/wf/workflow.hpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 587fedab..1db1ec2e 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -248,19 +248,21 @@ class AMSWorkflow if (phLoc != mlLoc) { // Copy out the result ResourceManager::copy(p_ml_acceptable, _p_ml_acceptable); - ResourceManager::deallocate(p_ml_acceptable, mlLoc); for (int i = 0; i < Outputs.size(); i++) { ResourceManager::copy(tmpOutputs[i], Outputs[i]); - + } // De-allocate temp data - for (int i = 0; i < Inputs.size(); i++) + for (int i = 0; i < Inputs.size(); i++) { ams::ResourceManager::deallocate(const_cast( tmpInputs[i]), mlLoc); - for (int i = 0; i < Outputs.size(); i++) + } + + for (int i = 0; i < Outputs.size(); i++) { ams::ResourceManager::deallocate((tmpOutputs[i]), mlLoc); + } + ams::ResourceManager::deallocate(p_ml_acceptable, mlLoc); - } } }