Modify the consistent hash

1. Modified the consistent hashing algorithm to fix a bug where only one real node was being accessed. 2. Updated the corresponding unit tests to reflect the changes."
baiyina · Sep 12, 2024 · bc373f9 · bc373f9
1 parent 7db4b55
commit bc373f9
Show file tree

Hide file tree

Showing 6 changed files with 162 additions and 43 deletions.
diff --git a/...va/com/crossoverjie/cim/common/route/algorithm/consistenthash/AbstractConsistentHash.java b/...va/com/crossoverjie/cim/common/route/algorithm/consistenthash/AbstractConsistentHash.java
@@ -21,6 +21,11 @@ public abstract class AbstractConsistentHash {
      */
     protected abstract void add(long key,String value);
 
+    /**
+     * Clear old data in the structure
+     */
+    protected abstract void clear();
+
     /**
      * 排序节点，数据结构自身支持排序可以不用重写
      */
@@ -41,6 +46,7 @@ protected void sort(){}
      */
     public String process(List<String> values,String key){
 
+        clear();
         for (String value : values) {
             add(hash(value), value);
         }

diff --git a/...om/crossoverjie/cim/common/route/algorithm/consistenthash/SortArrayMapConsistentHash.java b/...om/crossoverjie/cim/common/route/algorithm/consistenthash/SortArrayMapConsistentHash.java
@@ -20,8 +20,6 @@ public class SortArrayMapConsistentHash extends AbstractConsistentHash {
 
     @Override
     public void add(long key, String value) {
-        // fix https://github.com/crossoverJie/cim/issues/79
-        sortArrayMap.clear();
         for (int i = 0; i < VIRTUAL_NODE_SIZE; i++) {
             Long hash = super.hash("vir" + key + i);
             sortArrayMap.add(hash,value);
@@ -34,6 +32,11 @@ public void sort() {
         sortArrayMap.sort();
     }
 
+    @Override
+    protected void clear() {
+        sortArrayMap.clear();
+    }
+
     @Override
     public String getFirstNodeValue(String value) {
         long hash = super.hash(value);

diff --git a/...ava/com/crossoverjie/cim/common/route/algorithm/consistenthash/TreeMapConsistentHash.java b/...ava/com/crossoverjie/cim/common/route/algorithm/consistenthash/TreeMapConsistentHash.java
@@ -23,16 +23,18 @@ public class TreeMapConsistentHash extends AbstractConsistentHash {
 
     @Override
     public void add(long key, String value) {
-
-        // fix https://github.com/crossoverJie/cim/issues/79
-        treeMap.clear();
         for (int i = 0; i < VIRTUAL_NODE_SIZE; i++) {
             Long hash = super.hash("vir" + key + i);
             treeMap.put(hash,value);
         }
         treeMap.put(key, value);
     }
 
+    @Override
+    protected void clear() {
+        treeMap.clear();
+    }
+
     @Override
     public String getFirstNodeValue(String value) {
         long hash = super.hash(value);

diff --git a/...t/java/com/crossoverjie/cim/common/route/algorithm/consistenthash/RangeCheckTestUtil.java b/...t/java/com/crossoverjie/cim/common/route/algorithm/consistenthash/RangeCheckTestUtil.java
@@ -0,0 +1,15 @@
+package com.crossoverjie.cim.common.route.algorithm.consistenthash;
+
+import org.junit.Assert;
+
+/**
+ * @description: TODO
+ * @author: zhangguoa
+ * @date: 2024/9/12 9:58
+ * @project: cim
+ */
+public class RangeCheckTestUtil {
+    public static void assertInRange (int value, int l, int r) {
+        Assert.assertTrue(value >= l && value <= r);
+    }
+}
diff --git a/...rossoverjie/cim/common/route/algorithm/consistenthash/SortArrayMapConsistentHashTest.java b/...rossoverjie/cim/common/route/algorithm/consistenthash/SortArrayMapConsistentHashTest.java
@@ -1,10 +1,11 @@
 package com.crossoverjie.cim.common.route.algorithm.consistenthash;
 
+import com.crossoverjie.cim.common.data.construct.SortArrayMap;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.lang.reflect.Field;
+import java.util.*;
 
 public class SortArrayMapConsistentHashTest {
 
@@ -16,10 +17,11 @@ public void getFirstNodeValue() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"zhangsan");
-        System.out.println(process);
-        Assert.assertEquals("127.0.0.9",process);
-
+        String PROCESS = map.process(strings, "zhangsan");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "zhangsan");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
 
     @Test
@@ -30,10 +32,12 @@ public void getFirstNodeValue2() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"zhangsan2");
-        System.out.println(process);
 
-        Assert.assertEquals("127.0.0.9",process);
+        String PROCESS = map.process(strings,"zhangsan2");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "zhangsan2");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
 
     @Test
@@ -44,10 +48,11 @@ public void getFirstNodeValue3() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"1551253899106");
-
-        System.out.println(process);
-        Assert.assertEquals("127.0.0.9",process);
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
 
 
@@ -59,10 +64,12 @@ public void getFirstNodeValue4() {
         strings.add("45.78.28.220:9000:8081") ;
         strings.add("45.78.28.220:9100:9081") ;
 
-        String process = map.process(strings,"1551253899106");
 
-        System.out.println(process);
-        Assert.assertEquals("45.78.28.220:9100:9081",process);
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
     @Test
     public void getFirstNodeValue5() {
@@ -73,10 +80,11 @@ public void getFirstNodeValue5() {
         strings.add("45.78.28.220:9100:9081") ;
         strings.add("45.78.28.220:9100:10081") ;
 
-        String process = map.process(strings,"1551253899106");
-
-        System.out.println(process);
-        Assert.assertEquals("45.78.28.220:9100:10081",process);
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
 
     @Test
@@ -88,10 +96,11 @@ public void getFirstNodeValue6() {
         strings.add("45.78.28.220:9100:9081") ;
         strings.add("45.78.28.220:9100:10081") ;
 
-        String process = map.process(strings,"1551253899106");
-
-        System.out.println(process);
-        Assert.assertEquals("45.78.28.220:9100:10081",process);
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
     @Test
     public void getFirstNodeValue7() {
@@ -103,12 +112,49 @@ public void getFirstNodeValue7() {
         strings.add("45.78.28.220:9100:10081") ;
         strings.add("45.78.28.220:9100:00081") ;
 
-        String process = map.process(strings,"1551253899106");
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
+    }
+
+    @Test
+    public void getFirstNodeValue8() {
+        AbstractConsistentHash map = new SortArrayMapConsistentHash() ;
 
-        System.out.println(process);
-        Assert.assertEquals("45.78.28.220:9100:00081",process);
+        List<String> strings = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            strings.add("127.0.0." + i);
+        }
+        Set<String> processes = new HashSet<>();
+        for (int i = 0; i < 10; i++) {
+            String process = map.process(strings,"zhangsan" + i);
+            processes.add(process);
+        }
+        RangeCheckTestUtil.assertInRange(processes.size(), 2, 10);
     }
 
+    @Test
+    public void testVirtualNode() throws NoSuchFieldException, IllegalAccessException {
+        AbstractConsistentHash map = new SortArrayMapConsistentHash();
 
+        List<String> strings = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            strings.add("127.0.0." + i);
+        }
+
+        String process = map.process(strings,"zhangsan");
+        Field sortArrayMapField = SortArrayMapConsistentHash.class.getDeclaredField("sortArrayMap");
+        sortArrayMapField.setAccessible(true);
+        Field virtualNodeSizeField = SortArrayMapConsistentHash.class.getDeclaredField("VIRTUAL_NODE_SIZE");
+        virtualNodeSizeField.setAccessible(true);
+
+        SortArrayMap sortArrayMap = (SortArrayMap) sortArrayMapField.get(map);
+        int virtualNodeSize = (int) virtualNodeSizeField.get(map);
+
+        System.out.println("sortArrayMapSize = " + sortArrayMap.size() + "\n" + "virtualNodeSize = " + virtualNodeSize);
+        Assert.assertEquals(sortArrayMap.size(), (virtualNodeSize + 1) * 10);
+    }
 
 }
diff --git a/...com/crossoverjie/cim/common/route/algorithm/consistenthash/TreeMapConsistentHashTest.java b/...com/crossoverjie/cim/common/route/algorithm/consistenthash/TreeMapConsistentHashTest.java
@@ -3,11 +3,13 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.lang.reflect.Field;
+import java.util.*;
 
 public class TreeMapConsistentHashTest {
 
+
+
     @Test
     public void getFirstNodeValue() {
         AbstractConsistentHash map = new TreeMapConsistentHash() ;
@@ -16,9 +18,11 @@ public void getFirstNodeValue() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"zhangsan");
-        System.out.println(process);
-        Assert.assertEquals("127.0.0.9",process);
+        String PROCESS = map.process(strings, "zhangsan");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "zhangsan");
+            Assert.assertEquals(PROCESS, process);
+        }
     }
 
 
@@ -31,10 +35,14 @@ public void getFirstNodeValue2() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"zhangsan2");
-        System.out.println(process);
+        String PROCESS = map.process(strings,"zhangsan2");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "zhangsan2");
+            Assert.assertEquals(PROCESS, process);
+        }
+
 
-        Assert.assertEquals("127.0.0.9",process);
+//        Assert.assertEquals("127.0.0.9",process);
     }
 
 
@@ -46,9 +54,48 @@ public void getFirstNodeValue3() {
         for (int i = 0; i < 10; i++) {
             strings.add("127.0.0." + i) ;
         }
-        String process = map.process(strings,"1551253899106");
+        String PROCESS = map.process(strings,"1551253899106");
+        for (int i = 0; i < 100; i++) {
+            String process = map.process(strings, "1551253899106");
+            Assert.assertEquals(PROCESS, process);
+        }
+    }
+
+    @Test
+    public void getFirstNodeValue4() {
+        AbstractConsistentHash map = new TreeMapConsistentHash() ;
+
+        List<String> strings = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            strings.add("127.0.0." + i);
+        }
+        Set<String> processes = new HashSet<>();
+        for (int i = 0; i < 10; i++) {
+            String process = map.process(strings,"zhangsan" + i);
+            processes.add(process);
+        }
+        RangeCheckTestUtil.assertInRange(processes.size(), 2, 10);
+    }
+
+    @Test
+    public void testVirtualNode() throws NoSuchFieldException, IllegalAccessException {
+        AbstractConsistentHash map = new TreeMapConsistentHash();
+
+        List<String> strings = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            strings.add("127.0.0." + i);
+        }
+
+        String process = map.process(strings,"zhangsan");
+        Field treeMapField = TreeMapConsistentHash.class.getDeclaredField("treeMap");
+        treeMapField.setAccessible(true);
+        Field virtualNodeSizeField = TreeMapConsistentHash.class.getDeclaredField("VIRTUAL_NODE_SIZE");
+        virtualNodeSizeField.setAccessible(true);
+
+        TreeMap treeMap = (TreeMap) treeMapField.get(map);
+        int virtualNodeSize = (int) virtualNodeSizeField.get(map);
 
-        System.out.println(process);
-        Assert.assertEquals("127.0.0.9",process);
+        System.out.println("treeMapSize = " + treeMap.size() + "\n" + "virtualNodeSize = " + virtualNodeSize);
+        Assert.assertEquals(treeMap.size(), (virtualNodeSize + 1) * 10);
     }
 }