mpi: make local fast path faster

src/mpi/MpiWorld.cpp

@@ -516,9 +516,7 @@ void MpiWorld::send(int sendRank,
     m->set_messagetype(messageType);
 
     // Set up message data
-    if (count > 0 && buffer != nullptr) {
-        m->set_buffer(buffer, dataType->size * count);
-    }
+    bool mustSendData = count > 0 && buffer != nullptr;
 
     // Mock the message sending in tests
     if (faabric::util::isMockMode()) {
@@ -528,10 +526,18 @@ void MpiWorld::send(int sendRank,
 
     // Dispatch the message locally or globally
     if (isLocal) {
+        if (mustSendData) {
+            m->set_bufferptr((uint64_t)buffer);
+        }
+
         SPDLOG_TRACE(
           "MPI - send {} -> {} ({})", sendRank, recvRank, messageType);
         getLocalQueue(sendRank, recvRank)->enqueue(std::move(m));
     } else {
+        if (mustSendData) {
+            m->set_buffer(buffer, dataType->size * count);
+        }
+
         SPDLOG_TRACE(
           "MPI - send remote {} -> {} ({})", sendRank, recvRank, messageType);
         sendRemoteMpiMessage(otherHost, sendRank, recvRank, m);
@@ -596,10 +602,16 @@ void MpiWorld::doRecv(std::shared_ptr<MPIMessage>& m,
     assert(m->messagetype() == messageType);
     assert(m->count() <= count);
 
-    // TODO - avoid copy here
-    // Copy message data
+    const std::string otherHost = getHostForRank(m->destination());
+    bool isLocal = otherHost == thisHost;
+
     if (m->count() > 0) {
-        std::move(m->buffer().begin(), m->buffer().end(), buffer);
+        if (isLocal) {
+            std::memcpy(buffer, (void*)m->bufferptr(), count * dataType->size);
+        } else {
+            // TODO - avoid copy here
+            std::move(m->buffer().begin(), m->buffer().end(), buffer);
+        }
     }
 
     // Set status values if required

src/mpi/mpi.proto

@@ -26,5 +26,10 @@ message MPIMessage {
     int32 destination = 5;
     int32 type = 6;
     int32 count = 7;
-    bytes buffer = 8;
+
+    // For remote messaging
+    optional bytes buffer = 8;
+
+    // For local messaging
+    optional int64 bufferPtr = 9;
 }