1.0.12-SNAPSHOT

fix rdb parser bug
leonchen83 · Sep 23, 2016 · 150e5c0 · 150e5c0
1 parent 4b5dc91
commit 150e5c0
Show file tree

Hide file tree

Showing 28 changed files with 67 additions and 28 deletions.
diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java
@@ -26,7 +26,7 @@ public AbstractRdbParser(RedisInputStream in, AbstractReplicator replicator) {
         this.replicator = replicator;
     }
 
-    protected long rdbLoad() throws IOException, InterruptedException {
+    protected long rdbLoad(int version) throws IOException, InterruptedException {
         throw new UnsupportedOperationException("rdbLoad()");
     }
 
@@ -297,7 +297,7 @@ public static int zmElementLen(RedisInputStream in) throws IOException {
             } else if (len == 254) {
                 return in.readInt(4, false);
             } else {
-                return -1;
+                return len;
             }
         }
 

diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java
@@ -19,9 +19,9 @@ public Rdb6Parser(RedisInputStream in, AbstractReplicator replicator) {
         super(in, replicator);
     }
 
-    protected long rdbLoad() throws IOException, InterruptedException {
+    protected long rdbLoad(int version) throws IOException, InterruptedException {
         Db db = null;
-        long checksum;
+        long checksum = 0;
         /**
          * rdb
          */
@@ -104,7 +104,7 @@ protected long rdbLoad() throws IOException, InterruptedException {
                  * ----------------------------
                  */
                 case REDIS_RDB_OPCODE_EOF:
-                    checksum = in.readLong(8);
+                    if (version >= 5) checksum = in.readLong(8);
                     break loop;
                 default:
                     throw new AssertionError("Un-except value-type:" + type);
@@ -205,23 +205,24 @@ private KeyValuePair rdbLoadObject(int rdbtype) throws IOException {
                 int zmlen = AbstractRdbParser.LenHelper.zmlen(stream);
                 while (true) {
                     int zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream);
-                    if (zmEleLen == -1) {
-                        break;
+                    if (zmEleLen == 255) {
+                        o9.setValueRdbType(rdbtype);
+                        o9.setValue(map);
+                        return o9;
                     }
                     String field = AbstractRdbParser.StringHelper.str(stream, zmEleLen);
                     zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream);
+                    if (zmEleLen == 255) {
+                        o9.setValueRdbType(rdbtype);
+                        o9.setValue(map);
+                        return o9;
+                    }
                     int free = AbstractRdbParser.LenHelper.free(stream);
                     String value = AbstractRdbParser.StringHelper.str(stream, zmEleLen);
                     AbstractRdbParser.StringHelper.skip(stream, free);
                     map.put(field, value);
                 }
-                int zmend = AbstractRdbParser.LenHelper.zmend(stream);
-                if (zmend != 255) {
-                    throw new AssertionError("zmend expected 255 but " + zmend);
-                }
-                o9.setValueRdbType(rdbtype);
-                o9.setValue(map);
-                return o9;
+
             /*
              * |<encoding>| <length-of-contents>|              <contents>                           |
              * | 4 bytes  |            4 bytes  | 2 bytes lement| 4 bytes element | 8 bytes element |

diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java
@@ -19,9 +19,9 @@ public Rdb7Parser(RedisInputStream in, AbstractReplicator replicator) {
         super(in, replicator);
     }
 
-    protected long rdbLoad() throws IOException, InterruptedException {
+    protected long rdbLoad(int version) throws IOException, InterruptedException {
         Db db = null;
-        long checksum;
+        long checksum = 0;
         /**
          * rdb
          */
@@ -120,7 +120,7 @@ protected long rdbLoad() throws IOException, InterruptedException {
                  * ----------------------------
                  */
                 case REDIS_RDB_OPCODE_EOF:
-                    checksum = in.readLong(8);
+                    if (version >= 5) checksum = in.readLong(8);
                     break loop;
                 default:
                     throw new AssertionError("Un-except value-type:" + type);
@@ -221,23 +221,23 @@ private KeyValuePair rdbLoadObject(int rdbtype) throws IOException {
                 int zmlen = AbstractRdbParser.LenHelper.zmlen(stream);
                 while (true) {
                     int zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream);
-                    if (zmEleLen == -1) {
-                        break;
+                    if (zmEleLen == 255) {
+                        o9.setValueRdbType(rdbtype);
+                        o9.setValue(map);
+                        return o9;
                     }
                     String field = AbstractRdbParser.StringHelper.str(stream, zmEleLen);
                     zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream);
+                    if (zmEleLen == 255) {
+                        o9.setValueRdbType(rdbtype);
+                        o9.setValue(map);
+                        return o9;
+                    }
                     int free = AbstractRdbParser.LenHelper.free(stream);
                     String value = AbstractRdbParser.StringHelper.str(stream, zmEleLen);
                     AbstractRdbParser.StringHelper.skip(stream, free);
                     map.put(field, value);
                 }
-                int zmend = AbstractRdbParser.LenHelper.zmend(stream);
-                if (zmend != 255) {
-                    throw new AssertionError("zmend expected 255 but " + zmend);
-                }
-                o9.setValueRdbType(rdbtype);
-                o9.setValue(map);
-                return o9;
             /*
              * |<encoding>| <length-of-contents>|              <contents>                           |
              * | 4 bytes  |            4 bytes  | 2 bytes lement| 4 bytes element | 8 bytes element |

diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java
@@ -101,7 +101,7 @@ public long parse() throws IOException {
                     return in.total();
             }
             this.replicator.submitEvent(new PreFullSyncEvent());
-            long checksum = rdbParser.rdbLoad();
+            long checksum = rdbParser.rdbLoad(version);
             this.replicator.submitEvent(new PostFullSyncEvent(checksum));
             return in.total();
         } catch (InterruptedException e) {

diff --git a/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java b/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java
@@ -16,10 +16,47 @@
 
 package com.moilioncircle.redis.replicator.rdb;
 
+import com.moilioncircle.redis.replicator.Configuration;
+import com.moilioncircle.redis.replicator.RedisReplicator;
+import com.moilioncircle.redis.replicator.Replicator;
+import com.moilioncircle.redis.replicator.rdb.datatype.KeyValuePair;
 import junit.framework.TestCase;
+import org.junit.Test;
 
-public class RdbParserTest extends TestCase {
+public class RdbParserTest {
 
+    @Test
     public void testParse() throws Exception {
+        String[] resources = new String[]{"dictionary.rdb", "dumpV6.rdb", "dumpV7.rdb",
+                "easily_compressible_string_key.rdb", "empty_database.rdb",
+                "hash_as_ziplist.rdb", "integer_keys.rdb", "intset_16.rdb",
+                "intset_32.rdb", "intset_64.rdb", "keys_with_expiry.rdb",
+                "linkedlist.rdb", "multiple_databases.rdb",
+                "parser_filters.rdb", "rdb_version_5_with_checksum.rdb", "regular_set.rdb",
+                "regular_sorted_set.rdb", "sorted_set_as_ziplist.rdb", "uncompressible_string_keys.rdb",
+                "ziplist_that_compresses_easily.rdb", "ziplist_that_doesnt_compress.rdb",
+                "ziplist_with_integers.rdb", "zipmap_that_compresses_easily.rdb",
+                "zipmap_that_doesnt_compress.rdb", "zipmap_with_big_values.rdb"};
+        for (String resource : resources) {
+            template(resource);
+        }
+    }
+
+    public void template(String filename) {
+        try {
+            RedisReplicator replicator = new RedisReplicator(RdbParserTest.class.
+                    getClassLoader().getResourceAsStream(filename)
+                    , Configuration.defaultSetting());
+            replicator.addRdbListener(new RdbListener.Adaptor() {
+                @Override
+                public void handle(Replicator replicator, KeyValuePair<?> kv) {
+                    System.out.println(kv);
+                }
+            });
+            replicator.open();
+            Thread.sleep(4000);
+        } catch (Exception e) {
+            TestCase.fail();
+        }
     }
 }
diff --git a/src/test/resources/dictionary.rdb b/src/test/resources/dictionary.rdb
diff --git a/src/test/resources/easily_compressible_string_key.rdb b/src/test/resources/easily_compressible_string_key.rdb
diff --git a/src/test/resources/empty_database.rdb b/src/test/resources/empty_database.rdb
@@ -0,0 +1 @@
+REDIS0003�
diff --git a/src/test/resources/hash_as_ziplist.rdb b/src/test/resources/hash_as_ziplist.rdb
diff --git a/src/test/resources/integer_keys.rdb b/src/test/resources/integer_keys.rdb
diff --git a/src/test/resources/intset_16.rdb b/src/test/resources/intset_16.rdb
diff --git a/src/test/resources/intset_32.rdb b/src/test/resources/intset_32.rdb
diff --git a/src/test/resources/intset_64.rdb b/src/test/resources/intset_64.rdb
diff --git a/src/test/resources/keys_with_expiry.rdb b/src/test/resources/keys_with_expiry.rdb
diff --git a/src/test/resources/linkedlist.rdb b/src/test/resources/linkedlist.rdb
diff --git a/src/test/resources/multiple_databases.rdb b/src/test/resources/multiple_databases.rdb
diff --git a/src/test/resources/parser_filters.rdb b/src/test/resources/parser_filters.rdb
diff --git a/src/test/resources/rdb_version_5_with_checksum.rdb b/src/test/resources/rdb_version_5_with_checksum.rdb
diff --git a/src/test/resources/regular_set.rdb b/src/test/resources/regular_set.rdb
diff --git a/src/test/resources/regular_sorted_set.rdb b/src/test/resources/regular_sorted_set.rdb
diff --git a/src/test/resources/sorted_set_as_ziplist.rdb b/src/test/resources/sorted_set_as_ziplist.rdb
diff --git a/src/test/resources/uncompressible_string_keys.rdb b/src/test/resources/uncompressible_string_keys.rdb
diff --git a/src/test/resources/ziplist_that_compresses_easily.rdb b/src/test/resources/ziplist_that_compresses_easily.rdb
diff --git a/src/test/resources/ziplist_that_doesnt_compress.rdb b/src/test/resources/ziplist_that_doesnt_compress.rdb
diff --git a/src/test/resources/ziplist_with_integers.rdb b/src/test/resources/ziplist_with_integers.rdb
diff --git a/src/test/resources/zipmap_that_compresses_easily.rdb b/src/test/resources/zipmap_that_compresses_easily.rdb
diff --git a/src/test/resources/zipmap_that_doesnt_compress.rdb b/src/test/resources/zipmap_that_doesnt_compress.rdb
diff --git a/src/test/resources/zipmap_with_big_values.rdb b/src/test/resources/zipmap_with_big_values.rdb