Use "huge pages" when on unix-like operating systems

Makefile

@@ -12,6 +12,7 @@ LDFLAGS+= -Wall -lpthread -lssl -lcrypto
 
 LIBOBJECTS = \
 	./src/hashutil.o \
+	./src/memory.o
 
 HEADERS = $(wildcard src/*.h)
 ALIB = libcuckoofilter.a

benchmarks/Makefile

@@ -8,7 +8,7 @@ LDFLAGS+= -Wall -lpthread -lssl -lcrypto
 
 HEADERS = $(wildcard ../src/*.h) *.h
 
-SRC = ../src/hashutil.cc
+SRC = ../src/hashutil.cc ../src/memory.cc
 
 .PHONY: all
 

src/memory.cc

@@ -0,0 +1,71 @@
+#include "memory.h"
+
+#include <cstdlib>
+#include <cstring>
+
+#if defined(MMAP)
+
+#include <sys/mman.h>  // mmap, munmap
+#include <cerrno>      // errno
+#include <new>         // std::bad_alloc
+#include <stdexcept>   // std::runtime_error
+
+static constexpr uint64_t HUGE_PAGE_SIZE = ((uint64_t)1) << 21;
+
+// OVERAGE_LIMIT is how much wiggle room there is on allocating more
+// memory than specifically requested.
+static constexpr double OVERAGE_LIMIT = 0.05;
+
+#if defined(__linux__) && __linux__
+#define MMAP_ZERO_FILLED true
+#else
+#define MMAP_ZERO_FILLED false
+#endif  // __linux__
+
+#endif  // MMAP
+
+namespace cuckoofilter {
+
+void *Allocate(std::size_t bytes, std::size_t *actual_bytes) noexcept(!kMmap) {
+#if defined(MMAP)
+  const double overage =
+      static_cast<double>(HUGE_PAGE_SIZE - bytes % HUGE_PAGE_SIZE) /
+      static_cast<double>(bytes);
+  if (overage < OVERAGE_LIMIT) {
+    bytes = ((bytes + HUGE_PAGE_SIZE - 1) / HUGE_PAGE_SIZE) * HUGE_PAGE_SIZE;
+    *actual_bytes = bytes;
+    errno = 0;
+    void *result = mmap(NULL, bytes, PROT_READ | PROT_WRITE,
+                        MAP_PRIVATE | MAP_HUGETLB | MAP_ANONYMOUS, -1, 0);
+    if (MAP_FAILED == result) {
+      throw std::runtime_error(std::strerror(errno));
+    }
+    if (!MMAP_ZERO_FILLED) std::memset(result, 0, bytes);
+    return result;
+  }
+#endif  // MMAP
+  *actual_bytes = bytes;
+  void * result;
+  const int malloc_failed = posix_memalign(&result, 64, bytes);
+  if (malloc_failed) throw std::runtime_error(std::strerror(malloc_failed));
+  std::memset(result, 0, bytes);
+  return result;
+}
+
+void Deallocate(void *p, std::size_t bytes) noexcept(!kMmap) {
+#if defined(MMAP)
+  const double overage =
+      static_cast<double>(HUGE_PAGE_SIZE - bytes % HUGE_PAGE_SIZE) /
+      static_cast<double>(bytes);
+  if (overage < OVERAGE_LIMIT) {
+    bytes = ((bytes + HUGE_PAGE_SIZE - 1) / HUGE_PAGE_SIZE) * HUGE_PAGE_SIZE;
+    errno = 0;
+    const int fail = munmap(p, bytes);
+    if (fail != 0) throw std::runtime_error(std::strerror(errno));
+    return;
+  }
+#endif  // MMAP
+  std::free(p);
+}
+
+}  // namespace cuckoofilter

src/memory.h

@@ -0,0 +1,41 @@
+#ifndef CUCKOO_FILTER_MEMORY_H_
+#define CUCKOO_FILTER_MEMORY_H_
+
+// This file provides two functions dealing with memory
+// allocation. They abstract out complexities like allocating aligned
+// memory and using 2MB "huge pages" rather than the usual 4KB pages,
+// hopefully thereby reducing TLB misses.
+//
+// In benchmarking on 126M elements, this induced a <1.5% space
+// increase, a 9% decrease in the wall-clock time to run
+// bulk-insert-and-query.exe, as well as a 56% reduction in page
+// faults and a 99% reduction in dTLB misses.
+
+#include <cstddef>
+
+#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) &&                      \
+    (((defined(_BSD_SOURCE) || defined(_SVID_SOURCE)) && __GLIBC__ == 2 && \
+      __GLIBC_MINOR__ <= 19) ||                                            \
+     (defined(_DEFAULT_SOURCE) &&                                          \
+      ((__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 19))))
+
+#define MMAP
+
+#endif  // MMAP
+
+static constexpr bool kMmap =
+#if defined(MMAP)
+    true;
+#else
+    false;
+#endif  // MMAP
+
+namespace cuckoofilter {
+
+void *Allocate(std::size_t bytes, std::size_t *actual_bytes) noexcept(!kMmap);
+
+void Deallocate(void *p, std::size_t bytes) noexcept(!kMmap);
+
+}  // namespace cuckoofilter
+
+#endif // CUCKOO_FILTER_MEMORY_H_

src/packedtable.h

@@ -5,6 +5,7 @@
 #include <utility>
 
 #include "debug.h"
+#include "memory.h"
 #include "permencoding.h"
 #include "printutil.h"
 
@@ -20,6 +21,7 @@ class PackedTable {
 
   // using a pointer adds one more indirection
   size_t len_;
+  size_t bytes_;
   size_t num_buckets_;
   char *buckets_;
   PermEncoding perm_;
@@ -29,12 +31,11 @@ class PackedTable {
     // NOTE(binfan): use 7 extra bytes to avoid overrun as we
     // always read a uint64
     len_ = kBytesPerBucket * num_buckets_ + 7;
-    buckets_ = new char[len_];
-    memset(buckets_, 0, len_); 
+    buckets_ = reinterpret_cast<char *>(Allocate(len_, &bytes_));
   }
 
   ~PackedTable() { 
-    delete[] buckets_; 
+    Deallocate(buckets_, len_);
   }
 
   size_t NumBuckets() const {
@@ -46,7 +47,7 @@ class PackedTable {
   }
 
   size_t SizeInBytes() const { 
-    return len_; 
+    return bytes_;
   }
 
   std::string Info() const {

src/simd-block.h

@@ -21,6 +21,7 @@
 #include <immintrin.h>
 
 #include "hashutil.h"
+#include "memory.h"
 
 using uint32_t = ::std::uint32_t;
 using uint64_t = ::std::uint64_t;
@@ -41,6 +42,10 @@ class SimdBlockFilter {
   // log_num_buckets_ is the log (base 2) of the number of buckets in the directory:
   const int log_num_buckets_;
 
+  // The number of bytes returned by the allocation function; might be
+  // slightly more than sizeof(Bucket) * (1 << log_num_buckets_)
+  uint64_t actual_bytes_;
+
   // directory_mask_ is (1 << log_num_buckets_) - 1. It is precomputed in the contructor
   // for efficiency reasons:
   const uint32_t directory_mask_;
@@ -54,13 +59,14 @@ class SimdBlockFilter {
   explicit SimdBlockFilter(const int log_heap_space);
   SimdBlockFilter(SimdBlockFilter&& that)
     : log_num_buckets_(that.log_num_buckets_),
+      actual_bytes_(that.actual_bytes_),
       directory_mask_(that.directory_mask_),
       directory_(that.directory_),
       hasher_(that.hasher_) {}
   ~SimdBlockFilter() noexcept;
   void Add(const uint64_t key) noexcept;
   bool Find(const uint64_t key) const noexcept;
-  uint64_t SizeInBytes() const { return sizeof(Bucket) * (1ull << log_num_buckets_); }
+  uint64_t SizeInBytes() const { return actual_bytes_; }
 
  private:
   // A helper function for Insert()/Find(). Turns a 32-bit hash into a 256-bit Bucket
@@ -85,15 +91,14 @@ SimdBlockFilter<HashFamily>::SimdBlockFilter(const int log_heap_space)
     throw ::std::runtime_error("SimdBlockFilter does not work without AVX2 instructions");
   }
   const size_t alloc_size = 1ull << (log_num_buckets_ + LOG_BUCKET_BYTE_SIZE);
-  const int malloc_failed =
-      posix_memalign(reinterpret_cast<void**>(&directory_), 64, alloc_size);
-  if (malloc_failed) throw ::std::bad_alloc();
-  memset(directory_, 0, alloc_size);
+  directory_ = reinterpret_cast<Bucket *>(
+      cuckoofilter::Allocate(alloc_size, &actual_bytes_));
 }
 
 template<typename HashFamily>
 SimdBlockFilter<HashFamily>::~SimdBlockFilter() noexcept {
-  free(directory_);
+  const size_t alloc_size = 1ull << (log_num_buckets_ + LOG_BUCKET_BYTE_SIZE);
+  cuckoofilter::Deallocate(directory_, alloc_size);
   directory_ = nullptr;
 }
 

src/singletable.h

@@ -7,6 +7,7 @@
 
 #include "bitsutil.h"
 #include "debug.h"
+#include "memory.h"
 #include "printutil.h"
 
 namespace cuckoofilter {
@@ -30,23 +31,25 @@ class SingleTable {
   // using a pointer adds one more indirection
   Bucket *buckets_;
   size_t num_buckets_;
+  size_t bytes_;
 
  public:
   explicit SingleTable(const size_t num) : num_buckets_(num) {
-    buckets_ = new Bucket[num_buckets_ + kPaddingBuckets];
-    memset(buckets_, 0, kBytesPerBucket * (num_buckets_ + kPaddingBuckets));
+    const size_t bytes = sizeof(Bucket) * (num_buckets_ + kPaddingBuckets);
+    buckets_ = reinterpret_cast<Bucket *>(Allocate(bytes, &bytes_));
   }
 
-  ~SingleTable() { 
-    delete[] buckets_;
+  ~SingleTable() noexcept(!kMmap) {
+    const size_t bytes = sizeof(Bucket) * (num_buckets_ + kPaddingBuckets);
+    Deallocate(buckets_, bytes);
   }
 
   size_t NumBuckets() const {
     return num_buckets_;
   }
 
   size_t SizeInBytes() const { 
-    return kBytesPerBucket * num_buckets_; 
+    return bytes_;
   }
 
   size_t SizeInTags() const {