From 6a7b2c419e5b50d956676482a2088015449a828a Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Thu, 18 Jan 2024 11:33:00 +0900 Subject: [PATCH 1/4] Working --- Libplanet.Store/Trie/MerkleTrie.cs | 89 +++++++----------------------- Libplanet.Store/TrieStateStore.cs | 18 ++++-- 2 files changed, 34 insertions(+), 73 deletions(-) diff --git a/Libplanet.Store/Trie/MerkleTrie.cs b/Libplanet.Store/Trie/MerkleTrie.cs index 1c2650dfab3..3bd582daf33 100644 --- a/Libplanet.Store/Trie/MerkleTrie.cs +++ b/Libplanet.Store/Trie/MerkleTrie.cs @@ -192,62 +192,23 @@ internal IEnumerable> IterateHashNodes() yield break; } - var queue = - new Queue<(KeyBytes Key, byte[] Value, ImmutableArray Path)>(); - switch (Root) - { - case ValueNode valueNode: - var value = _codec.Encode(valueNode.ToBencodex()); - var key = new KeyBytes(HashDigest.DeriveFrom(value).ByteArray); - yield return (key, value); - yield break; - - case HashNode hashNode: - key = new KeyBytes(hashNode.HashDigest.ToByteArray()); - queue.Enqueue((key, KeyValueStore.Get(key), ImmutableArray.Empty)); - break; - - case FullNode _: - case ShortNode _: - value = _codec.Encode(Root.ToBencodex()); - key = new KeyBytes(HashDigest.DeriveFrom(value).ByteArray); - queue.Enqueue((key, value, ImmutableArray.Empty)); - break; - } - - bool GuessValueNodeByPath(in ImmutableArray path) - { - if (path.Length < 2) - { - return false; - } - - bool isStartedWithUnderbar = (path[0] << 4) + path[1] == '_'; - - bool isStatePath = !isStartedWithUnderbar && - path.Length == Address.Size * 2 * 2; - return isStatePath; - } + var queue = new Queue(); + queue.Enqueue(Root); while (queue.Count > 0) { - (KeyBytes key, byte[] value, ImmutableArray path) = - queue.Dequeue(); - - // It assumes every length of value nodes is same with Address' hexadecimal - // string's hexadecimal string's size. - bool isValueNode = GuessValueNodeByPath(path); - - yield return (key, value); - - if (isValueNode) - { - continue; - } - - var node = NodeDecoder.Decode(_codec.Decode(value), NodeDecoder.AnyNodeType); - if (isValueNode) + INode node = queue.Dequeue(); + if (node is HashNode dequeuedHashNode) { + var storedKey = new KeyBytes(dequeuedHashNode.HashDigest.ByteArray); + var storedValue = KeyValueStore.Get(storedKey); + var intermediateEncoding = _codec.Decode(storedValue); + queue.Enqueue( + NodeDecoder.Decode( + intermediateEncoding, + NodeDecoder.HashEmbeddedNodeType) ?? + throw new NullReferenceException()); + yield return (storedKey, storedValue); continue; } @@ -257,33 +218,23 @@ bool GuessValueNodeByPath(in ImmutableArray path) foreach (int index in Enumerable.Range(0, FullNode.ChildrenCount - 1)) { INode? child = fullNode.Children[index]; - if (child is HashNode hashNode) + if (child is HashNode childHashNode) { - key = new KeyBytes(hashNode.HashDigest.ByteArray); - value = KeyValueStore.Get(key); - queue.Enqueue((key, value, path.Add((byte)index))); + queue.Enqueue(childHashNode); } } - switch (fullNode.Value) + if (fullNode.Value is HashNode fullNodeValueHashNode) { - case HashNode hashNode: - key = new KeyBytes(hashNode.HashDigest.ByteArray); - value = KeyValueStore.Get(key); - queue.Enqueue((key, value, path)); - break; + queue.Enqueue(fullNodeValueHashNode); } break; case ShortNode shortNode: - switch (shortNode.Value) + if (shortNode.Value is HashNode shortNodeValueHashNode) { - case HashNode hashNode: - key = new KeyBytes(hashNode.HashDigest.ByteArray); - value = KeyValueStore.Get(key); - queue.Enqueue((key, value, path.AddRange(shortNode.Key.ByteArray))); - break; + queue.Enqueue(shortNodeValueHashNode); } break; @@ -291,7 +242,7 @@ bool GuessValueNodeByPath(in ImmutableArray path) case ValueNode _: break; - default: + case HashNode _: throw new InvalidOperationException(); } } diff --git a/Libplanet.Store/TrieStateStore.cs b/Libplanet.Store/TrieStateStore.cs index 62efeb5db50..1f859979832 100644 --- a/Libplanet.Store/TrieStateStore.cs +++ b/Libplanet.Store/TrieStateStore.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; @@ -93,29 +94,38 @@ public void PruneStates(IImmutableSet> survivingStateRootHash /// /// The state root hashes of states to copy. /// The target state store to copy state root hashes. + /// Thrown when a state root cannot be found for + /// any of given . public void CopyStates( IImmutableSet> stateRootHashes, TrieStateStore targetStateStore) { IKeyValueStore targetKeyValueStore = targetStateStore.StateKeyValueStore; var stopwatch = new Stopwatch(); + long count = 0; _logger.Verbose("Started {MethodName}()", nameof(CopyStates)); stopwatch.Start(); foreach (HashDigest stateRootHash in stateRootHashes) { - var stateTrie = new MerkleTrie( - StateKeyValueStore, - new HashNode(stateRootHash)); + var stateTrie = (MerkleTrie)GetStateRoot(stateRootHash); + if (!stateTrie.Recorded) + { + throw new ArgumentException( + $"Failed to find a state root for given state root hash {stateRootHash}."); + } foreach (var (key, value) in stateTrie.IterateNodeKeyValuePairs()) { targetKeyValueStore.Set(key, value); + count++; } } stopwatch.Stop(); _logger.Debug( - "Finished to copy all states {ElapsedMilliseconds} ms", + "Finished copying all states with {Count} key value pairs " + + "in {ElapsedMilliseconds} ms", + count, stopwatch.ElapsedMilliseconds); _logger.Verbose("Finished {MethodName}()", nameof(CopyStates)); } From e412805b25a4896f004588e0de59c00eeff4ab7b Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Thu, 18 Jan 2024 13:31:49 +0900 Subject: [PATCH 2/4] Refactored tests --- Libplanet.Tests/Store/TrieStateStoreTest.cs | 42 ++++++++++++--------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/Libplanet.Tests/Store/TrieStateStoreTest.cs b/Libplanet.Tests/Store/TrieStateStoreTest.cs index 4c38a56662c..d37621cd758 100644 --- a/Libplanet.Tests/Store/TrieStateStoreTest.cs +++ b/Libplanet.Tests/Store/TrieStateStoreTest.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Security.Cryptography; @@ -108,27 +110,25 @@ public void PruneStates() [Fact] public void CopyStates() { - var values = ImmutableDictionary.Empty - .Add(new KeyBytes("foo"), (Binary)GetRandomBytes(4096)) - .Add( - new KeyBytes("bar"), - (Text)ByteUtil.Hex(GetRandomBytes(2048))) - .Add(new KeyBytes("baz"), (Bencodex.Types.Boolean)false) - .Add(new KeyBytes("qux"), Bencodex.Types.Dictionary.Empty) - .Add( - new KeyBytes("zzz"), - Bencodex.Types.Dictionary.Empty - .Add("binary", GetRandomBytes(4096)) - .Add("text", ByteUtil.Hex(GetRandomBytes(2048)))); - var stateStore = new TrieStateStore(_stateKeyValueStore); - IKeyValueStore targetStateKeyValueStore = new MemoryKeyValueStore(); var targetStateStore = new TrieStateStore(targetStateKeyValueStore); - ITrie trie = stateStore.Commit( - values.Aggregate( - stateStore.GetStateRoot(null), - (prev, kv) => prev.Set(kv.Key, kv.Value))); + Random random = new Random(); + List<(KeyBytes, IValue)> kvs = Enumerable.Range(0, 1_000) + .Select(_ => + ( + new KeyBytes(GetRandomBytes(random.Next(20))), + (IValue)new Binary(GetRandomBytes(20)) + )) + .ToList(); + + ITrie trie = stateStore.GetStateRoot(null); + foreach (var kv in kvs) + { + trie = trie.Set(kv.Item1, kv.Item2); + } + + trie = stateStore.Commit(trie); int prevStatesCount = _stateKeyValueStore.ListKeys().Count(); _stateKeyValueStore.Set( @@ -149,6 +149,12 @@ public void CopyStates() // FIXME: Bencodex fingerprints also should be tracked. // https://github.com/planetarium/libplanet/issues/1653 Assert.Equal(prevStatesCount, targetStateKeyValueStore.ListKeys().Count()); + Assert.Equal( + trie.IterateNodes().Count(), + targetStateStore.GetStateRoot(trie.Hash).IterateNodes().Count()); + Assert.Equal( + trie.IterateValues().Count(), + targetStateStore.GetStateRoot(trie.Hash).IterateValues().Count()); } [Fact] From ce54ebd07727d7c940c818cce9178253c4a8cb82 Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Thu, 18 Jan 2024 13:40:17 +0900 Subject: [PATCH 3/4] Cleanup and some documentation --- Libplanet.Store/Trie/MerkleTrie.cs | 11 ++++++++++- Libplanet.Store/TrieStateStore.cs | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Libplanet.Store/Trie/MerkleTrie.cs b/Libplanet.Store/Trie/MerkleTrie.cs index 3bd582daf33..3bc6bf67cd9 100644 --- a/Libplanet.Store/Trie/MerkleTrie.cs +++ b/Libplanet.Store/Trie/MerkleTrie.cs @@ -185,7 +185,16 @@ internal IEnumerable> IterateHashNodes() .Select(pair => ((HashNode)pair.Node).HashDigest); } - internal IEnumerable<(KeyBytes Key, byte[] Value)> IterateNodeKeyValuePairs() + /// + /// Iterates over and pairs stored + /// necessary to fully represent this . + /// + /// An of all and + /// pairs stored necessary to fully represent + /// this . + /// Thrown when a + /// is encountered that can't be decoded into an . + internal IEnumerable<(KeyBytes Key, byte[] Value)> IterateKeyValuePairs() { if (Root is null) { diff --git a/Libplanet.Store/TrieStateStore.cs b/Libplanet.Store/TrieStateStore.cs index 1f859979832..68127cc4087 100644 --- a/Libplanet.Store/TrieStateStore.cs +++ b/Libplanet.Store/TrieStateStore.cs @@ -114,7 +114,7 @@ public void CopyStates( $"Failed to find a state root for given state root hash {stateRootHash}."); } - foreach (var (key, value) in stateTrie.IterateNodeKeyValuePairs()) + foreach (var (key, value) in stateTrie.IterateKeyValuePairs()) { targetKeyValueStore.Set(key, value); count++; From c61248ddc720a28cea18f5dfda28d65709052b7e Mon Sep 17 00:00:00 2001 From: Say Cheong Date: Thu, 18 Jan 2024 13:44:05 +0900 Subject: [PATCH 4/4] Tweaked tests --- Libplanet.Tests/Store/TrieStateStoreTest.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Libplanet.Tests/Store/TrieStateStoreTest.cs b/Libplanet.Tests/Store/TrieStateStoreTest.cs index d37621cd758..c2099d50a41 100644 --- a/Libplanet.Tests/Store/TrieStateStoreTest.cs +++ b/Libplanet.Tests/Store/TrieStateStoreTest.cs @@ -131,11 +131,12 @@ public void CopyStates() trie = stateStore.Commit(trie); int prevStatesCount = _stateKeyValueStore.ListKeys().Count(); + // NOTE: Avoid possible collision of KeyBytes, just in case. _stateKeyValueStore.Set( - new KeyBytes("alpha"), + new KeyBytes(GetRandomBytes(30)), ByteUtil.ParseHex("00")); _stateKeyValueStore.Set( - new KeyBytes("beta"), + new KeyBytes(GetRandomBytes(40)), ByteUtil.ParseHex("00")); Assert.Equal(prevStatesCount + 2, _stateKeyValueStore.ListKeys().Count());