[resolve #18] Add C++ allocator (partial)

CMakeLists.txt

@@ -22,7 +22,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
 string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -g -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ")
-string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra ")
+string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra")
 
 ###
 # Mandatory: devices configuration file.
@@ -127,6 +127,7 @@ target_link_libraries(rdmalib PRIVATE cereal)
 # client library
 ###
 file(GLOB rdmalib_files "rfaas/lib/*.cpp")
+message(STATUS "rdmalib_files ${rdmalib_files}")
 add_library(rfaaslib STATIC ${rdmalib_files})
 add_dependencies(rfaaslib spdlog)
 add_dependencies(rfaaslib cereal)

benchmarks/warm_benchmark.cpp

@@ -13,6 +13,7 @@
 
 #include <rfaas/executor.hpp>
 #include <rfaas/resources.hpp>
+#include <rfaas/rdma_allocator.hpp>
 
 #include "warm_benchmark.hpp"
 #include "settings.hpp"
@@ -68,27 +69,35 @@ int main(int argc, char ** argv)
   }
 
   // FIXME: move me to a memory allocator
-  rdmalib::Buffer<char> in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size);
-  in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE);
-  out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-  memset(in.data(), 0, opts.input_size);
-  for(int i = 0; i < opts.input_size; ++i) {
-    ((char*)in.data())[i] = 1;
+
+  rfaas::RdmaAllocator<rdmalib::Buffer<char> > rdmaAllocator(executor);
+  rdmalib::Buffer<char>* in = rdmaAllocator.allocate(opts.input_size, IBV_ACCESS_LOCAL_WRITE,
+                                   rdmalib::functions::Submission::DATA_HEADER_SIZE);
+  rdmalib::Buffer<char>* out = rdmaAllocator.allocate(opts.input_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+  // rdmalib::Buffer<char> in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size);
+  // in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE);
+  // out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+
+  // TODO: Since the for loop writes a value of 1 to each byte of the in buffer,
+  //       it overwrites all bytes previously set to 0 by the memset() function.
+  memset(in->data(), 0, opts.input_size);
+  for (int i = 0; i < opts.input_size; ++i) {
+    ((char *) in->data())[i] = 1;
   }
 
   rdmalib::Benchmarker<1> benchmarker{settings.benchmark.repetitions};
   spdlog::info("Warmups begin");
   for(int i = 0; i < settings.benchmark.warmup_repetitions; ++i) {
     SPDLOG_DEBUG("Submit warm {}", i);
-    executor.execute(opts.fname, in, out);
+    executor.execute(opts.fname, *in, *out);
   }
   spdlog::info("Warmups completed");
 
   // Start actual measurements
   for(int i = 0; i < settings.benchmark.repetitions;) {
     benchmarker.start();
     SPDLOG_DEBUG("Submit execution {}", i);
-    auto ret = executor.execute(opts.fname, in, out);
+    auto ret = executor.execute(opts.fname, *in, *out);
     if(std::get<0>(ret)) {
       SPDLOG_DEBUG("Finished execution {} out of {}", i, settings.benchmark.repetitions);
       benchmarker.end(0);
@@ -108,7 +117,7 @@ int main(int argc, char ** argv)
 
   printf("Data: ");
   for(int i = 0; i < std::min(100, opts.input_size); ++i)
-    printf("%d ", ((char*)out.data())[i]);
+    printf("%d ", ((char*)out->data())[i]);
   printf("\n");
 
   return 0;

rfaas/include/rfaas/rdma_allocator.hpp

@@ -0,0 +1,58 @@
+//
+// Created by mou on 4/2/23.
+//
+
+#ifndef __RFAAS_RDMA_ALLOCATOR_HPP__
+#define __RFAAS_RDMA_ALLOCATOR_HPP__
+
+#include <cstddef>
+#include <rdmalib/buffer.hpp>
+#include <rfaas/executor.hpp>
+
+namespace rfaas {
+  template<typename T>
+  class RdmaAllocator {
+  public:
+    typedef T value_type;
+
+    inline explicit RdmaAllocator(const executor &executor) noexcept: _executor(executor) {}
+
+    template<class U>
+    constexpr RdmaAllocator(const RdmaAllocator<U> &) noexcept {}
+
+    [[nodiscard]] inline T *allocate(const std::size_t &size, const int &access, int header = 0) {
+      if (size > std::numeric_limits<std::size_t>::max() / sizeof(T))
+        throw std::bad_array_new_length();
+
+      // Maybe we could directly call the memset function here
+      if (auto buffer = new rdmalib::Buffer<char>(size, header)) {
+        report(buffer, size);
+        buffer->register_memory(_executor._state.pd(), access);
+        return buffer;
+      }
+      throw std::bad_alloc();
+    }
+
+    inline void deallocate(T *p, std::size_t size) noexcept {
+      report(p, size, 0);
+      std::free(p);
+    }
+
+  private:
+    const executor &_executor;
+
+    void report(T *p, std::size_t n, bool alloc = true) const {
+      std::cout << (alloc ? "Alloc: " : "Dealloc: ") << sizeof(T) * n
+                << " bytes at " << std::hex << std::showbase
+                << reinterpret_cast<void *>(p) << std::dec << '\n';
+    }
+  };
+
+  template<class T, class U>
+  bool operator==(const RdmaAllocator<T> &, const RdmaAllocator<U> &) { return true; }
+
+  template<class T, class U>
+  bool operator!=(const RdmaAllocator<T> &, const RdmaAllocator<U> &) { return false; }
+}
+
+#endif //__RFAAS_RDMA_ALLOCATOR_HPP__