key value inline

ART/N.cpp

@@ -649,9 +649,9 @@ Leaf::Leaf(const Key *k) : BaseNode(NTypes::Leaf) {
     key_len = k->key_len;
     val_len = k->val_len;
 #ifdef KEY_INLINE
-    key = k->key; // compare to store the key, new an array will decrease
-                  // 30% performance
-    fkey = (uint8_t *)&key;
+    // have allocate the memory for kv
+    memcpy(kv, k->fkey, key_len);
+    memcpy(kv + key_len, (void *)k->value, val_len);
 #else
     // allocate from NVM for variable key
     fkey = new (alloc_new_node_from_size(key_len)) uint8_t[key_len];
@@ -672,10 +672,15 @@ Leaf::Leaf(uint8_t *key_, size_t key_len_, char *value_, size_t val_len_)
     : BaseNode(NTypes::Leaf) {
     key_len = key_len_;
     val_len = val_len_;
+#ifdef KEY_INLINE
+    memcpy(kv, key_, key_len);
+    memcpy(kv + key_len, value_, val_len);
+#else
     fkey = key_; // no need to alloc a new key, key_ is persistent
     value = new (alloc_new_node_from_size(val_len)) char[val_len];
     memcpy(value, (void *)value_, val_len);
     flush_data((void *)value, val_len);
+#endif
 }
 
 void N::helpFlush(std::atomic<N *> *n) {
@@ -1211,6 +1216,12 @@ void N::rebuild_node(N *node, std::set<std::pair<uint64_t, size_t>> &rs) {
         // leaf node
 #ifdef RECLAIM_MEMORY
         Leaf *leaf = N::getLeaf(node);
+#ifdef KEY_INLINE
+        size_t size =
+            size_align(sizeof(Leaf) + leaf->key_len + leaf->val_len, 64);
+        //        size = convert_power_two(size);
+        rs.insert(std::make_pair((uint64_t)leaf, size));
+#else
         NTypes type = leaf->type;
         size_t size = size_align(get_node_size(type), 64);
         //        size = convert_power_two(size);
@@ -1225,6 +1236,8 @@ void N::rebuild_node(N *node, std::set<std::pair<uint64_t, size_t>> &rs) {
         size = leaf->val_len;
         //        size = convert_power_two(size);
         rs.insert(std::make_pair((uint64_t)(leaf->value), size));
+#endif // KEY_INLINE
+
 #endif
         return;
     }

ART/N.h

@@ -200,10 +200,14 @@ class Leaf : public BaseNode {
   public:
     size_t key_len;
     size_t val_len;
-    uint64_t key;
-    // variable key
+//    uint64_t key;
+// variable key
+#ifdef KEY_INLINE
+    char kv[0]; // append key and value
+#else
     uint8_t *fkey;
     char *value;
+#endif
 
   public:
     Leaf(const Key *k);
@@ -215,11 +219,31 @@ class Leaf : public BaseNode {
     virtual ~Leaf() {}
 
     bool checkKey(const Key *k) const {
+#ifdef KEY_INLINE
+        if (key_len == k->getKeyLen() && memcmp(kv, k->fkey, key_len) == 0)
+            return true;
+        return false;
+#else
         if (key_len == k->getKeyLen() && memcmp(fkey, k->fkey, key_len) == 0)
             return true;
         return false;
+#endif
     }
     size_t getKeyLen() const { return key_len; }
+    char *GetKey() {
+#ifdef KEY_INLINE
+        return kv;
+#else
+        return (char *)fkey;
+#endif
+    }
+    char *GetValue() {
+#ifdef KEY_INLINE
+        return kv + key_len;
+#else
+        return value;
+#endif
+    }
 } __attribute__((aligned(64)));
 
 static constexpr uint32_t maxStoredPrefixLength = 4;

ART/N16.cpp

@@ -151,7 +151,8 @@ bool N16::insert(uint8_t key, N *n, bool flush) {
 
     if (flush) {
         uint64_t oldp = (1ull << 56) | ((uint64_t)compactCount << 48);
-        old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+        old_pointer.store(oldp,
+                          std::memory_order_seq_cst); // store the old version
     }
 
     children[compactCount].store(n, std::memory_order_seq_cst);

ART/N256.cpp

@@ -97,7 +97,8 @@ void N256::deleteChildren() {
 bool N256::insert(uint8_t key, N *val, bool flush) {
     if (flush) {
         uint64_t oldp = (1ull << 56) | ((uint64_t)key << 48);
-        old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+        old_pointer.store(oldp,
+                          std::memory_order_seq_cst); // store the old version
     }
 
     children[key].store(val, std::memory_order_seq_cst);

ART/N4.cpp

@@ -152,12 +152,12 @@ bool N4::insert(uint8_t key, N *n, bool flush) {
     keys[compactCount].store(key, std::memory_order_seq_cst);
     if (flush)
         flush_data((void *)&keys[compactCount], sizeof(std::atomic<uint8_t>));
-    //        clflush((char *)&keys[compactCount], sizeof(uint8_t), true, true);
 
     // record the old version and index
     if (flush) {
         uint64_t oldp = (1ull << 56) | ((uint64_t)compactCount << 48);
-        old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+        old_pointer.store(oldp,
+                          std::memory_order_seq_cst); // store the old version
     }
 
     // modify the pointer
@@ -166,7 +166,9 @@ bool N4::insert(uint8_t key, N *n, bool flush) {
     // flush the new pointer and clear the old version
     if (flush) {
         flush_data((void *)&children[compactCount], sizeof(std::atomic<N *>));
-        old_pointer.store(0, std::memory_order_seq_cst); // after persisting, clear the old version
+        old_pointer.store(0,
+                          std::memory_order_seq_cst); // after persisting, clear
+                                                      // the old version
     }
 
     compactCount++;
@@ -181,7 +183,8 @@ void N4::change(uint8_t key, N *val) {
 
             uint64_t oldp = (1ull << 56) | ((uint64_t)i << 48) |
                             ((uint64_t)children[i].load() & ((1ull << 48) - 1));
-            old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+            old_pointer.store(
+                oldp, std::memory_order_seq_cst); // store the old version
 
             children[i].store(val, std::memory_order_seq_cst);
             flush_data((void *)&children[i], sizeof(std::atomic<N *>));
@@ -229,7 +232,8 @@ bool N4::remove(uint8_t k, bool force, bool flush) {
 
             uint64_t oldp = (1ull << 56) | ((uint64_t)i << 48) |
                             ((uint64_t)children[i].load() & ((1ull << 48) - 1));
-            old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+            old_pointer.store(
+                oldp, std::memory_order_seq_cst); // store the old version
 
             children[i].store(nullptr, std::memory_order_seq_cst);
             flush_data((void *)&children[i], sizeof(std::atomic<N *>));

ART/N48.cpp

@@ -122,7 +122,8 @@ bool N48::insert(uint8_t key, N *n, bool flush) {
     // record the old version and index
     if (flush) {
         uint64_t oldp = (1ull << 56) | ((uint64_t)compactCount << 48);
-        old_pointer.store(oldp, std::memory_order_seq_cst); // store the old version
+        old_pointer.store(oldp,
+                          std::memory_order_seq_cst); // store the old version
     }
 
     // modify the pointer
@@ -131,7 +132,9 @@ bool N48::insert(uint8_t key, N *n, bool flush) {
     // flush the new pointer and clear the old version
     if (flush) {
         flush_data((void *)&children[compactCount], sizeof(std::atomic<N *>));
-        old_pointer.store(0, std::memory_order_seq_cst); // after persisting, clear the old version
+        old_pointer.store(0,
+                          std::memory_order_seq_cst); // after persisting, clear
+                                                      // the old version
     }
 
     compactCount++;
@@ -251,7 +254,7 @@ void N48::getChildren(uint8_t start, uint8_t end,
 
             if (valid && ind == index) {
                 if ((N *)p != nullptr) {
-                    children[childrenCount] = std::make_tuple(i, (N*)p);
+                    children[childrenCount] = std::make_tuple(i, (N *)p);
                     childrenCount++;
                 }
             } else {

ART/Tree.cpp

@@ -927,6 +927,22 @@ Tree::~Tree() {
     close_nvm_mgr();
 }
 
+// allocate a leaf and persist it
+Leaf *Tree::allocLeaf(const Key *k) const {
+#ifdef KEY_INLINE
+    Leaf *newLeaf =
+        new (alloc_new_node_from_size(sizeof(Leaf) + k->key_len + k->val_len))
+            Leaf(k);
+    flush_data((void *)newLeaf, sizeof(Leaf) + k->key_len + k->val_len);
+    return newLeaf;
+#else
+    Leaf *newLeaf =
+        new (alloc_new_node_from_type(NTypes::Leaf)) Leaf(k); // not persist
+    flush_data((void *)newLeaf, sizeof(Leaf));
+    return newLeaf;
+#endif
+}
+
 Leaf *Tree::lookup(const Key *k) const {
     // enter a new epoch
     EpochGuard NewEpoch;
@@ -1039,11 +1055,7 @@ typename Tree::OperationResults Tree::update(const Key *k) const {
                     return OperationResults::NotFound;
                 }
                 //
-                Leaf *newleaf = new (alloc_new_node_from_type(NTypes::Leaf))
-                    Leaf(leaf->fkey, k->key_len, (char *)k->value, k->val_len);
-                //                    std::cout<<(int)(((BaseNode
-                //                    *)newleaf)->type)<<"\n";
-                flush_data((void *)newleaf, sizeof(Leaf));
+                Leaf *newleaf = allocLeaf(k);
                 //
                 N::change(node, nodeKey, N::setLeaf(newleaf));
                 node->writeUnlock();
@@ -1248,8 +1260,13 @@ bool Tree::lookupRange(const Key *start, const Key *end, const Key *continueKey,
 
     if (toContinue != nullptr) {
         Key *newkey = new Key();
+#ifdef KEY_INLINE
+        newkey->Init((char *)toContinue->GetKey(), toContinue->key_len,
+                     toContinue->GetValue(), toContinue->val_len);
+#else
         newkey->Init((char *)toContinue->fkey, toContinue->key_len,
                      toContinue->value, toContinue->val_len);
+#endif
         continueKey = newkey;
         return true;
     } else {
@@ -1301,11 +1318,7 @@ typename Tree::OperationResults Tree::insert(const Key *k) {
                 N4(nextLevel, prefi); // not persist
 
             // 2)  add node and (tid, *k) as children
-            Leaf *newLeaf = new (alloc_new_node_from_type(NTypes::Leaf))
-                Leaf(k); // not persist
-            flush_data((void *)newLeaf, sizeof(Leaf));
-            //            N::clflush((char *)newLeaf, sizeof(Leaf), true,
-            //                       true); // persist leaf and key pointer
+            Leaf *newLeaf = allocLeaf(k);
 
             // not persist
             newNode->insert(k->fkey[nextLevel], N::setLeaf(newLeaf), false);
@@ -1353,10 +1366,7 @@ typename Tree::OperationResults Tree::insert(const Key *k) {
             if (needRestart)
                 goto restart;
 
-            Leaf *newLeaf =
-                new (alloc_new_node_from_type(NTypes::Leaf)) Leaf(k);
-            flush_data((void *)newLeaf, sizeof(Leaf));
-            //            N::clflush((char *)newLeaf, sizeof(Leaf), true, true);
+            Leaf *newLeaf = allocLeaf(k);
 
             N::insertAndUnlock(node, parentNode, parentKey, nodeKey,
                                N::setLeaf(newLeaf), needRestart);
@@ -1378,12 +1388,21 @@ typename Tree::OperationResults Tree::insert(const Key *k) {
             // but if I want to insert a prefix of this key, i also need to
             // insert successfully
             uint32_t prefixLength = 0;
+#ifdef KEY_INLINE
+            while (level + prefixLength <
+                       std::min(k->getKeyLen(), key->getKeyLen()) &&
+                   key->kv[level + prefixLength] ==
+                       k->fkey[level + prefixLength]) {
+                prefixLength++;
+            }
+#else
             while (level + prefixLength <
                        std::min(k->getKeyLen(), key->getKeyLen()) &&
                    key->fkey[level + prefixLength] ==
                        k->fkey[level + prefixLength]) {
                 prefixLength++;
             }
+#endif
             // equal
             if (k->getKeyLen() == key->getKeyLen() &&
                 level + prefixLength == k->getKeyLen()) {
@@ -1395,15 +1414,18 @@ typename Tree::OperationResults Tree::insert(const Key *k) {
             // substring
 
             auto n4 = new (alloc_new_node_from_type(NTypes::N4))
-                N4(level + prefixLength, &k->fkey[level], prefixLength);
-            Leaf *newLeaf =
-                new (alloc_new_node_from_type(NTypes::Leaf)) Leaf(k);
-            flush_data((void *)newLeaf, sizeof(Leaf));
+                N4(level + prefixLength, &k->fkey[level],
+                   prefixLength); // not persist
+            Leaf *newLeaf = allocLeaf(k);
             //            N::clflush((char *)newLeaf, sizeof(Leaf), true, true);
 
             n4->insert(k->fkey[level + prefixLength], N::setLeaf(newLeaf),
                        false);
+#ifdef KEY_INLINE
+            n4->insert(key->kv[level + prefixLength], nextNode, false);
+#else
             n4->insert(key->fkey[level + prefixLength], nextNode, false);
+#endif
             flush_data((void *)n4, sizeof(N4));
             //            N::clflush((char *)n4, sizeof(N4), true, true);
 
@@ -1592,8 +1614,13 @@ Tree::checkPrefixPessimistic(N *n, const Key *k, uint32_t &level,
             Leaf *kr = N::getAnyChildTid(n);
             p.prefixCount = discrimination;
             for (uint32_t i = 0;
-                 i < std::min(discrimination, maxStoredPrefixLength); i++)
+                 i < std::min(discrimination, maxStoredPrefixLength); i++) {
+#ifdef KEY_INLINE
+                p.prefix[i] = kr->kv[level + i];
+#else
                 p.prefix[i] = kr->fkey[level + i];
+#endif
+            }
             n->setPrefix(p.prefix, p.prefixCount, true);
             n->writeUnlock();
         }
@@ -1618,8 +1645,13 @@ Tree::checkPrefixPessimistic(N *n, const Key *k, uint32_t &level,
                 kt = N::getAnyChildTid(n);
                 load_flag = true;
             }
+#ifdef KEY_INLINE
+            uint8_t curKey = i >= maxStoredPrefixLength ? (uint8_t)kt->kv[level]
+                                                        : p.prefix[i];
+#else
             uint8_t curKey =
                 i >= maxStoredPrefixLength ? kt->fkey[level] : p.prefix[i];
+#endif
             if (curKey != k->fkey[level]) {
                 nonMatchingKey = curKey;
                 if (p.prefixCount > maxStoredPrefixLength) {
@@ -1630,7 +1662,12 @@ Tree::checkPrefixPessimistic(N *n, const Key *k, uint32_t &level,
                          j < std::min((p.prefixCount - (level - prevLevel) - 1),
                                       maxStoredPrefixLength);
                          ++j) {
+#ifdef KEY_INLINE
+                        nonMatchingPrefix.prefix[j] =
+                            (uint8_t)kt->kv[level + j + 1];
+#else
                         nonMatchingPrefix.prefix[j] = kt->fkey[level + j + 1];
+#endif
                     }
                 } else {
                     for (uint32_t j = 0; j < p.prefixCount - i - 1; ++j) {
@@ -1664,8 +1701,13 @@ Tree::checkPrefixCompare(const N *n, const Key *k, uint32_t &level) {
             uint8_t kLevel =
                 (k->getKeyLen() > level) ? k->fkey[level] : (uint8_t)0;
 
+#ifdef KEY_INLINE
+            uint8_t curKey = i >= maxStoredPrefixLength ? (uint8_t)kt->kv[level]
+                                                        : p.prefix[i];
+#else
             uint8_t curKey =
                 i >= maxStoredPrefixLength ? kt->fkey[level] : p.prefix[i];
+#endif
             if (curKey < kLevel) {
                 return PCCompareResults::Smaller;
             } else if (curKey > kLevel) {
@@ -1700,8 +1742,13 @@ typename Tree::PCEqualsResults Tree::checkPrefixEquals(const N *n,
             uint8_t endLevel =
                 (end->getKeyLen() > level) ? end->fkey[level] : (uint8_t)0;
 
+#ifdef KEY_INLINE
+            uint8_t curKey = i >= maxStoredPrefixLength ? (uint8_t)kt->kv[level]
+                                                        : p.prefix[i];
+#else
             uint8_t curKey =
                 i >= maxStoredPrefixLength ? kt->fkey[level] : p.prefix[i];
+#endif
             if (curKey > startLevel && curKey < endLevel) {
                 return PCEqualsResults::Contained;
             } else if (curKey < startLevel || curKey > endLevel) {

ART/Tree.h

@@ -147,6 +147,8 @@ class Tree {
     OperationResults insert(const Key *k);
 
     OperationResults remove(const Key *k);
+
+    Leaf *allocLeaf(const Key *k) const;
 } __attribute__((aligned(64)));
 
 #ifdef CHECK_COUNT