diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java index f1fae3ca..b0901977 100644 --- a/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java +++ b/src/main/java/com/moilioncircle/redis/replicator/rdb/AbstractRdbParser.java @@ -26,7 +26,7 @@ public AbstractRdbParser(RedisInputStream in, AbstractReplicator replicator) { this.replicator = replicator; } - protected long rdbLoad() throws IOException, InterruptedException { + protected long rdbLoad(int version) throws IOException, InterruptedException { throw new UnsupportedOperationException("rdbLoad()"); } @@ -297,7 +297,7 @@ public static int zmElementLen(RedisInputStream in) throws IOException { } else if (len == 254) { return in.readInt(4, false); } else { - return -1; + return len; } } diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java index 443c0830..55bda870 100644 --- a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java +++ b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb6Parser.java @@ -19,9 +19,9 @@ public Rdb6Parser(RedisInputStream in, AbstractReplicator replicator) { super(in, replicator); } - protected long rdbLoad() throws IOException, InterruptedException { + protected long rdbLoad(int version) throws IOException, InterruptedException { Db db = null; - long checksum; + long checksum = 0; /** * rdb */ @@ -104,7 +104,7 @@ protected long rdbLoad() throws IOException, InterruptedException { * ---------------------------- */ case REDIS_RDB_OPCODE_EOF: - checksum = in.readLong(8); + if (version >= 5) checksum = in.readLong(8); break loop; default: throw new AssertionError("Un-except value-type:" + type); @@ -205,23 +205,24 @@ private KeyValuePair rdbLoadObject(int rdbtype) throws IOException { int zmlen = AbstractRdbParser.LenHelper.zmlen(stream); while (true) { int zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream); - if (zmEleLen == -1) { - break; + if (zmEleLen == 255) { + o9.setValueRdbType(rdbtype); + o9.setValue(map); + return o9; } String field = AbstractRdbParser.StringHelper.str(stream, zmEleLen); zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream); + if (zmEleLen == 255) { + o9.setValueRdbType(rdbtype); + o9.setValue(map); + return o9; + } int free = AbstractRdbParser.LenHelper.free(stream); String value = AbstractRdbParser.StringHelper.str(stream, zmEleLen); AbstractRdbParser.StringHelper.skip(stream, free); map.put(field, value); } - int zmend = AbstractRdbParser.LenHelper.zmend(stream); - if (zmend != 255) { - throw new AssertionError("zmend expected 255 but " + zmend); - } - o9.setValueRdbType(rdbtype); - o9.setValue(map); - return o9; + /* * || | | * | 4 bytes | 4 bytes | 2 bytes lement| 4 bytes element | 8 bytes element | diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java index 1a6442e3..4e43da84 100644 --- a/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java +++ b/src/main/java/com/moilioncircle/redis/replicator/rdb/Rdb7Parser.java @@ -19,9 +19,9 @@ public Rdb7Parser(RedisInputStream in, AbstractReplicator replicator) { super(in, replicator); } - protected long rdbLoad() throws IOException, InterruptedException { + protected long rdbLoad(int version) throws IOException, InterruptedException { Db db = null; - long checksum; + long checksum = 0; /** * rdb */ @@ -120,7 +120,7 @@ protected long rdbLoad() throws IOException, InterruptedException { * ---------------------------- */ case REDIS_RDB_OPCODE_EOF: - checksum = in.readLong(8); + if (version >= 5) checksum = in.readLong(8); break loop; default: throw new AssertionError("Un-except value-type:" + type); @@ -221,23 +221,23 @@ private KeyValuePair rdbLoadObject(int rdbtype) throws IOException { int zmlen = AbstractRdbParser.LenHelper.zmlen(stream); while (true) { int zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream); - if (zmEleLen == -1) { - break; + if (zmEleLen == 255) { + o9.setValueRdbType(rdbtype); + o9.setValue(map); + return o9; } String field = AbstractRdbParser.StringHelper.str(stream, zmEleLen); zmEleLen = AbstractRdbParser.LenHelper.zmElementLen(stream); + if (zmEleLen == 255) { + o9.setValueRdbType(rdbtype); + o9.setValue(map); + return o9; + } int free = AbstractRdbParser.LenHelper.free(stream); String value = AbstractRdbParser.StringHelper.str(stream, zmEleLen); AbstractRdbParser.StringHelper.skip(stream, free); map.put(field, value); } - int zmend = AbstractRdbParser.LenHelper.zmend(stream); - if (zmend != 255) { - throw new AssertionError("zmend expected 255 but " + zmend); - } - o9.setValueRdbType(rdbtype); - o9.setValue(map); - return o9; /* * || | | * | 4 bytes | 4 bytes | 2 bytes lement| 4 bytes element | 8 bytes element | diff --git a/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java b/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java index a8a82834..8edd75db 100644 --- a/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java +++ b/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java @@ -101,7 +101,7 @@ public long parse() throws IOException { return in.total(); } this.replicator.submitEvent(new PreFullSyncEvent()); - long checksum = rdbParser.rdbLoad(); + long checksum = rdbParser.rdbLoad(version); this.replicator.submitEvent(new PostFullSyncEvent(checksum)); return in.total(); } catch (InterruptedException e) { diff --git a/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java b/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java index 4e92c6df..6237818f 100644 --- a/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java +++ b/src/test/java/com/moilioncircle/redis/replicator/rdb/RdbParserTest.java @@ -16,10 +16,47 @@ package com.moilioncircle.redis.replicator.rdb; +import com.moilioncircle.redis.replicator.Configuration; +import com.moilioncircle.redis.replicator.RedisReplicator; +import com.moilioncircle.redis.replicator.Replicator; +import com.moilioncircle.redis.replicator.rdb.datatype.KeyValuePair; import junit.framework.TestCase; +import org.junit.Test; -public class RdbParserTest extends TestCase { +public class RdbParserTest { + @Test public void testParse() throws Exception { + String[] resources = new String[]{"dictionary.rdb", "dumpV6.rdb", "dumpV7.rdb", + "easily_compressible_string_key.rdb", "empty_database.rdb", + "hash_as_ziplist.rdb", "integer_keys.rdb", "intset_16.rdb", + "intset_32.rdb", "intset_64.rdb", "keys_with_expiry.rdb", + "linkedlist.rdb", "multiple_databases.rdb", + "parser_filters.rdb", "rdb_version_5_with_checksum.rdb", "regular_set.rdb", + "regular_sorted_set.rdb", "sorted_set_as_ziplist.rdb", "uncompressible_string_keys.rdb", + "ziplist_that_compresses_easily.rdb", "ziplist_that_doesnt_compress.rdb", + "ziplist_with_integers.rdb", "zipmap_that_compresses_easily.rdb", + "zipmap_that_doesnt_compress.rdb", "zipmap_with_big_values.rdb"}; + for (String resource : resources) { + template(resource); + } + } + + public void template(String filename) { + try { + RedisReplicator replicator = new RedisReplicator(RdbParserTest.class. + getClassLoader().getResourceAsStream(filename) + , Configuration.defaultSetting()); + replicator.addRdbListener(new RdbListener.Adaptor() { + @Override + public void handle(Replicator replicator, KeyValuePair kv) { + System.out.println(kv); + } + }); + replicator.open(); + Thread.sleep(4000); + } catch (Exception e) { + TestCase.fail(); + } } } \ No newline at end of file diff --git a/src/test/resources/dictionary.rdb b/src/test/resources/dictionary.rdb new file mode 100644 index 00000000..e0ef5289 Binary files /dev/null and b/src/test/resources/dictionary.rdb differ diff --git a/src/test/resources/easily_compressible_string_key.rdb b/src/test/resources/easily_compressible_string_key.rdb new file mode 100644 index 00000000..73fca694 Binary files /dev/null and b/src/test/resources/easily_compressible_string_key.rdb differ diff --git a/src/test/resources/empty_database.rdb b/src/test/resources/empty_database.rdb new file mode 100644 index 00000000..1f48fb8f --- /dev/null +++ b/src/test/resources/empty_database.rdb @@ -0,0 +1 @@ +REDIS0003ÿ \ No newline at end of file diff --git a/src/test/resources/hash_as_ziplist.rdb b/src/test/resources/hash_as_ziplist.rdb new file mode 100644 index 00000000..c46e52ec Binary files /dev/null and b/src/test/resources/hash_as_ziplist.rdb differ diff --git a/src/test/resources/integer_keys.rdb b/src/test/resources/integer_keys.rdb new file mode 100644 index 00000000..ae740a64 Binary files /dev/null and b/src/test/resources/integer_keys.rdb differ diff --git a/src/test/resources/intset_16.rdb b/src/test/resources/intset_16.rdb new file mode 100644 index 00000000..a950132c Binary files /dev/null and b/src/test/resources/intset_16.rdb differ diff --git a/src/test/resources/intset_32.rdb b/src/test/resources/intset_32.rdb new file mode 100644 index 00000000..39c96097 Binary files /dev/null and b/src/test/resources/intset_32.rdb differ diff --git a/src/test/resources/intset_64.rdb b/src/test/resources/intset_64.rdb new file mode 100644 index 00000000..70f56c05 Binary files /dev/null and b/src/test/resources/intset_64.rdb differ diff --git a/src/test/resources/keys_with_expiry.rdb b/src/test/resources/keys_with_expiry.rdb new file mode 100644 index 00000000..6414be59 Binary files /dev/null and b/src/test/resources/keys_with_expiry.rdb differ diff --git a/src/test/resources/linkedlist.rdb b/src/test/resources/linkedlist.rdb new file mode 100644 index 00000000..70cd20f5 Binary files /dev/null and b/src/test/resources/linkedlist.rdb differ diff --git a/src/test/resources/multiple_databases.rdb b/src/test/resources/multiple_databases.rdb new file mode 100644 index 00000000..027b82d8 Binary files /dev/null and b/src/test/resources/multiple_databases.rdb differ diff --git a/src/test/resources/parser_filters.rdb b/src/test/resources/parser_filters.rdb new file mode 100644 index 00000000..a3eaaeb1 Binary files /dev/null and b/src/test/resources/parser_filters.rdb differ diff --git a/src/test/resources/rdb_version_5_with_checksum.rdb b/src/test/resources/rdb_version_5_with_checksum.rdb new file mode 100644 index 00000000..0a533c31 Binary files /dev/null and b/src/test/resources/rdb_version_5_with_checksum.rdb differ diff --git a/src/test/resources/regular_set.rdb b/src/test/resources/regular_set.rdb new file mode 100644 index 00000000..bbd844f0 Binary files /dev/null and b/src/test/resources/regular_set.rdb differ diff --git a/src/test/resources/regular_sorted_set.rdb b/src/test/resources/regular_sorted_set.rdb new file mode 100644 index 00000000..8204d6c9 Binary files /dev/null and b/src/test/resources/regular_sorted_set.rdb differ diff --git a/src/test/resources/sorted_set_as_ziplist.rdb b/src/test/resources/sorted_set_as_ziplist.rdb new file mode 100644 index 00000000..37cc2199 Binary files /dev/null and b/src/test/resources/sorted_set_as_ziplist.rdb differ diff --git a/src/test/resources/uncompressible_string_keys.rdb b/src/test/resources/uncompressible_string_keys.rdb new file mode 100644 index 00000000..276f334e Binary files /dev/null and b/src/test/resources/uncompressible_string_keys.rdb differ diff --git a/src/test/resources/ziplist_that_compresses_easily.rdb b/src/test/resources/ziplist_that_compresses_easily.rdb new file mode 100644 index 00000000..7fcb8e85 Binary files /dev/null and b/src/test/resources/ziplist_that_compresses_easily.rdb differ diff --git a/src/test/resources/ziplist_that_doesnt_compress.rdb b/src/test/resources/ziplist_that_doesnt_compress.rdb new file mode 100644 index 00000000..060a6cca Binary files /dev/null and b/src/test/resources/ziplist_that_doesnt_compress.rdb differ diff --git a/src/test/resources/ziplist_with_integers.rdb b/src/test/resources/ziplist_with_integers.rdb new file mode 100644 index 00000000..18d0594f Binary files /dev/null and b/src/test/resources/ziplist_with_integers.rdb differ diff --git a/src/test/resources/zipmap_that_compresses_easily.rdb b/src/test/resources/zipmap_that_compresses_easily.rdb new file mode 100644 index 00000000..d8d3cffd Binary files /dev/null and b/src/test/resources/zipmap_that_compresses_easily.rdb differ diff --git a/src/test/resources/zipmap_that_doesnt_compress.rdb b/src/test/resources/zipmap_that_doesnt_compress.rdb new file mode 100644 index 00000000..b556aa59 Binary files /dev/null and b/src/test/resources/zipmap_that_doesnt_compress.rdb differ diff --git a/src/test/resources/zipmap_with_big_values.rdb b/src/test/resources/zipmap_with_big_values.rdb new file mode 100644 index 00000000..89677fe1 Binary files /dev/null and b/src/test/resources/zipmap_with_big_values.rdb differ