3.5.4

leonchen83 · Mar 15, 2021 · 9260742 · 9260742
1 parent 6439524
commit 9260742
Show file tree

Hide file tree

Showing 9 changed files with 103 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+### 3.5.4
+
+Fix `DumpRdbValueVisitor` lzf compress bug.  
+
 ### 3.5.3
 
 `DumpRdbValueVisitor` support downgrade from redis 6.2 to 2.8.  

diff --git a/README.md b/README.md
@@ -84,7 +84,7 @@ redis 2.6 - 6.2
     <dependency>
         <groupId>com.moilioncircle</groupId>
         <artifactId>redis-replicator</artifactId>
-        <version>3.5.3</version>
+        <version>3.5.4</version>
     </dependency>
 ```
 

diff --git a/README.zh_CN.md b/README.zh_CN.md
@@ -90,7 +90,7 @@ redis 2.6 - 6.2
     <dependency>
         <groupId>com.moilioncircle</groupId>
         <artifactId>redis-replicator</artifactId>
-        <version>3.5.3</version>
+        <version>3.5.4</version>
     </dependency>
 ```
 

diff --git a/pom.xml b/pom.xml
@@ -22,7 +22,7 @@
 
     <groupId>com.moilioncircle</groupId>
     <artifactId>redis-replicator</artifactId>
-    <version>3.5.3</version>
+    <version>3.5.4</version>
 
     <name>redis-replicator</name>
     <description>

diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/BaseRdbEncoder.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/BaseRdbEncoder.java
@@ -170,13 +170,18 @@ public void rdbSaveBinaryDoubleValue(double value, OutputStream out) throws IOEx
      * @see BaseRdbParser#rdbLoadEncodedStringObject()
      */
     public void rdbSaveEncodedStringObject(ByteArray bytes, OutputStream out) throws IOException {
-        int type = (RDB_ENCVAL << 6) | RDB_ENC_LZF;
-        out.write(type);
-        ByteArray compressed = new ByteArray(bytes.length() + 4);
+        // at least compress 4 bytes
+        ByteArray compressed = new ByteArray(bytes.length() - 4);
         long length = Lzf.encode(bytes, bytes.length(), compressed, 0);
-        rdbSaveLen(length, out);
-        rdbSaveLen(bytes.length(), out);
-        out.write(compressed.first(), 0, (int) length);
+        if (length <= 0) {
+            rdbSavePlainStringObject(bytes, out);
+        } else {
+            int type = (RDB_ENCVAL << 6) | RDB_ENC_LZF;
+            out.write(type);
+            rdbSaveLen(length, out);
+            rdbSaveLen(bytes.length(), out);
+            out.write(compressed.first(), 0, (int) length);
+        }
     }
 
     /**

diff --git a/src/main/java/com/moilioncircle/redis/replicator/util/Lzf.java b/src/main/java/com/moilioncircle/redis/replicator/util/Lzf.java
@@ -248,11 +248,15 @@ private static void decode(ByteArray in, long inPos, ByteArray out, long outPos,
 
     public static ByteArray encode(ByteArray bytes) {
         long len = bytes.length();
-        ByteArray out = new ByteArray(len + 4);
+        ByteArray out = new ByteArray(len - 4);
         len = encode(bytes, len, out, 0);
-        ByteArray r = new ByteArray(len);
-        ByteArray.arraycopy(out, 0, r, 0, len);
-        return r;
+        if (len <= 0) {
+            return bytes;
+        } else {
+            ByteArray r = new ByteArray(len);
+            ByteArray.arraycopy(out, 0, r, 0, len);
+            return r;
+        }
     }
 
     public static long encode(ByteArray in, long inLen, ByteArray out, long outPos) {
@@ -286,6 +290,11 @@ public static long encode(ByteArray in, long inLen, ByteArray out, long outPos)
                 if (maxLen > MAX_REF) {
                     maxLen = MAX_REF;
                 }
+                if (outPos + 3 + 1 >= out.length()) {
+                    int c = literals == 0 ? 1 : 0;
+                    if (outPos - c + 3 + 1 >= out.length())
+                        return 0;
+                }
                 if (literals == 0) {
                     // multiple back-references,
                     // so there is no literal run control byte
@@ -321,6 +330,9 @@ public static long encode(ByteArray in, long inLen, ByteArray out, long outPos)
                 hashTab[hash(future)] = inPos++;
             } else {
                 // copy one byte from input to output as part of literal
+                if (outPos >= out.length()) {
+                    return 0;
+                }
                 out.set(outPos++, in.get(inPos++));
                 literals++;
                 // at the end of this literal chunk, write the length
@@ -334,6 +346,11 @@ public static long encode(ByteArray in, long inLen, ByteArray out, long outPos)
                 }
             }
         }
+
+        if (outPos + 3 > out.length()) {
+            return 0;
+        }
+
         // write the remaining few bytes as literals
         while (inPos < inLen) {
             out.set(outPos++, in.get(inPos++));

diff --git a/src/test/java/com/moilioncircle/redis/replicator/rdb/BaseRdbEncoderTest.java b/src/test/java/com/moilioncircle/redis/replicator/rdb/BaseRdbEncoderTest.java
@@ -19,12 +19,14 @@
 import static com.moilioncircle.redis.replicator.Constants.RDB_LOAD_ENC;
 import static java.lang.Double.NEGATIVE_INFINITY;
 import static java.lang.Double.POSITIVE_INFINITY;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
 
 import org.junit.Test;
 
@@ -62,6 +64,22 @@ public void testRdbGenericSaveStringObject() throws IOException {
 		}
 	}
 
+	@Test
+	public void testLzf() throws IOException {
+		for (int i = 0; i < 1000; i++) {
+			int length = ThreadLocalRandom.current().nextInt(50000) + 20;
+			byte[] value = new byte[length];
+			ThreadLocalRandom.current().nextBytes(value);
+			BaseRdbEncoder encoder = new BaseRdbEncoder();
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			encoder.rdbGenericSaveStringObject(new ByteArray(value), out);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			BaseRdbParser parser = new BaseRdbParser(new RedisInputStream(in));
+			byte[] bytes = parser.rdbGenericLoadStringObject(RDB_LOAD_ENC).first();
+			assertArrayEquals(value, bytes);
+		}
+	}
+
 	@Test
 	public void testRdbSaveDoubleValue() throws IOException {
 		{

diff --git a/src/test/java/com/moilioncircle/redis/replicator/util/CompressLZF.java b/src/test/java/com/moilioncircle/redis/replicator/util/CompressLZF.java
@@ -103,6 +103,11 @@ public int compress(byte[] in, int inLen, byte[] out, int outPos) {
                 if (maxLen > MAX_REF) {
                     maxLen = MAX_REF;
                 }
+                if (outPos + 3 + 1 >= out.length) {
+                    int c = literals == 0 ? 1 : 0;
+                    if (outPos - c + 3 + 1 >= out.length)
+                        return 0;
+                }
                 if (literals == 0) {
                     // multiple back-references,
                     // so there is no literal run control byte
@@ -138,6 +143,9 @@ public int compress(byte[] in, int inLen, byte[] out, int outPos) {
                 hashTab[hash(future)] = inPos++;
             } else {
                 // copy one byte from input to output as part of literal
+                if (outPos >= out.length) {
+                    return 0;
+                }
                 out[outPos++] = in[inPos++];
                 literals++;
                 // at the end of this literal chunk, write the length
@@ -151,6 +159,11 @@ public int compress(byte[] in, int inLen, byte[] out, int outPos) {
                 }
             }
         }
+
+        if (outPos + 3 > out.length) {
+            return 0;
+        }
+
         // write the remaining few bytes as literals
         while (inPos < inLen) {
             out[outPos++] = in[inPos++];

diff --git a/src/test/java/com/moilioncircle/redis/replicator/util/LzfTest.java b/src/test/java/com/moilioncircle/redis/replicator/util/LzfTest.java
@@ -19,9 +19,11 @@
 import org.junit.Test;
 
 import java.io.InputStream;
+import java.util.concurrent.ThreadLocalRandom;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author Leon Chen
@@ -74,6 +76,15 @@ public void encode() throws Exception {
             assertArrayEquals(out, out1);
         }
 
+        {
+            String str = "thisisalongstringthatcancompressbylzfthisisalongstringthatcancompressbylzf";
+            byte[] out = compress(str.getBytes());
+            byte[] out1 = compress1(str.getBytes());
+            assertTrue(str.getBytes().length > out.length);
+            assertEquals(out.length, out1.length);
+            assertArrayEquals(out, out1);
+        }
+
         {
             String str = "abcdsklafjslfjfd;sfdklafjlsafjslfjasl;fkjdsalfjasfjlas;dkfjalsvlasfkal;sj";
             byte[] out = compress(str.getBytes());
@@ -102,14 +113,32 @@ public void encode() throws Exception {
             assertArrayEquals(out, out1);
         }
     }
+
+    @Test
+    public void test1() {
+        for (int i = 0; i < 1000; i++) {
+            int length = ThreadLocalRandom.current().nextInt(50000) + 20;
+            byte[] value = new byte[length];
+            ThreadLocalRandom.current().nextBytes(value);
+            byte[] out = compress(value);
+            byte[] out1 = compress1(value);
+            assertEquals(out.length, out1.length);
+            assertArrayEquals(out, out1);
+        }
+    }
 
     private byte[] compress(byte[] in) {
         CompressLZF c = new CompressLZF();
-        byte[] compressed = new byte[in.length + 4];
+        byte[] compressed = new byte[in.length -4];
         int idx = c.compress(in, in.length, compressed, 0);
-        byte[] out = new byte[idx];
-        System.arraycopy(compressed, 0, out, 0, out.length);
-        return out;
+        if (idx <= 0) {
+            return in;
+        } else {
+            byte[] out = new byte[idx];
+            System.arraycopy(compressed, 0, out, 0, out.length);
+            return out;
+        }
+
     }
 
     private byte[] compress1(byte[] in) {