Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix getdata response for preleases #7331

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Fix getdata response for preleases #7331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5a3d132
Send back seednode version in get data response back to client
alvasw Dec 21, 2024
4698fa2
Filter for known items in HistoricalDataStoreService.getMapSinceVersi…
alvasw Dec 21, 2024
0a42bf6
Remove redundant Map creation in HistoricalDataStoreService.getMapSin…
alvasw Dec 21, 2024
e15a34f
Cap items if requesterVersion is null in HistoricalDataStoreService.g…
alvasw Dec 21, 2024
97330ba
Cap items if requesterVersion is newer in HistoricalDataStoreService.…
alvasw Dec 21, 2024
ddfa14a
Set wasPersistableNetworkPayloadsTruncated if requester Version newer…
alvasw Dec 21, 2024
da7cd47
Include responders missing hashes from our historical data store
alvasw Dec 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@
import com.google.common.base.Joiner;
import com.google.common.base.Optional;

import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

import java.lang.management.ManagementFactory;

Expand Down Expand Up @@ -100,7 +102,8 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) {
// Data
GetInventoryRequest getInventoryRequest = (GetInventoryRequest) networkEnvelope;
Map<InventoryItem, Integer> dataObjects = new HashMap<>();
p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion()).values().stream()
p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion(), Collections.emptySet(), new AtomicBoolean(false))
.values().stream()
.map(e -> e.getClass().getSimpleName())
.forEach(className -> addClassNameToMap(dataObjects, className));
p2PDataStorage.getMap().values().stream()
Expand Down
172 changes: 158 additions & 14 deletions core/src/test/java/bisq/core/account/witness/AccountAgeWitnessStorageServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@

import java.io.File;

import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -23,6 +27,7 @@


public class AccountAgeWitnessStorageServiceTest {
private final AtomicBoolean payloadsTruncated = new AtomicBoolean(false);
private DummyHistoricalDataStoreService storageService;

@BeforeEach
Expand All @@ -34,8 +39,9 @@ void setup(@TempDir Path tempDirPath) {
@Test
void emptyStore() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion(Version.VERSION);
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet(), payloadsTruncated);
assertThat(mapSinceVersion, is(anEmptyMap()));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
Expand All @@ -44,7 +50,7 @@ void testOnlyLiveData() {
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 2);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion(Version.VERSION);
storageService.getMapSinceVersion(Version.VERSION, Collections.emptySet(), payloadsTruncated);

P2PDataStorage.ByteArray firstByteArray = new P2PDataStorage.ByteArray(new byte[]{0});
P2PDataStorage.ByteArray secondByteArray = new P2PDataStorage.ByteArray(new byte[]{1});
Expand All @@ -54,6 +60,7 @@ void testOnlyLiveData() {
secondByteArray, liveDataMap.get(secondByteArray));

assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
Expand All @@ -67,21 +74,22 @@ void testOnlyStoreData() {
storageService.setStoresByVersion(storeByVersion);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.7.0");
storageService.getMapSinceVersion("1.7.0", Collections.emptySet(), payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(versionStore.getMap());
assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
void testRequesterVersionNull() {
void testRequesterVersionNullCappedLiveData() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessToMap(liveDataMap);
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 150);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 100);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 200);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 300);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
Expand All @@ -90,14 +98,45 @@ void testRequesterVersionNull() {
storageService.setStoresByVersion(storeByVersion);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion(null);
storageService.getMapSinceVersion(null, Collections.emptySet(), payloadsTruncated);

int mapSize = mapSinceVersion.size();
assertThat(mapSize, is(100));

for (Map.Entry<P2PDataStorage.ByteArray, PersistableNetworkPayload> entry : mapSinceVersion.entrySet()) {
var expected = liveDataMap.get(entry.getKey());
assertThat(entry.getValue(), is(expected));
}

assertThat(payloadsTruncated.get(), is(false));
}

@Test
void testRequesterUncappedData() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 150);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 2_000);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 3_000);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
"1.9.0", secondVersionStore);

storageService.setStoresByVersion(storeByVersion);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.2.3", Collections.emptySet(), payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>();
expected.putAll(liveDataMap);
expected.putAll(firstVersionStore.getMap());
expected.putAll(secondVersionStore.getMap());

assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
Expand All @@ -117,24 +156,25 @@ void testRequesterVersionIsOlder() {
storageService.setStoresByVersion(storeByVersion);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.8.5");
storageService.getMapSinceVersion("1.8.5", Collections.emptySet(), payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>();
expected.putAll(liveDataMap);
expected.putAll(secondVersionStore.getMap());

assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
void testRequesterVersionIsNewer() {
void testRequesterHasHistoricalDataStores() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessToMap(liveDataMap);
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 150);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(1, 100);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(1, 200);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(1, 300);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
Expand All @@ -143,9 +183,113 @@ void testRequesterVersionIsNewer() {
storageService.setStoresByVersion(storeByVersion);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.9.5");
storageService.getMapSinceVersion("1.9.5", Collections.emptySet(), payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(liveDataMap);
assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
void testRequesterVersionIsNewerCapped() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 150);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 200);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory
.createAccountAgeWitnessStore(1, 300);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
"1.9.0", secondVersionStore);

storageService.setStoresByVersion(storeByVersion);

String version = Version.VERSION;
String higherVersion = Version.getMajorVersion(version) + "." +
Version.getMinorVersion(version) + "." + (Version.getPatchVersion(version) + 1);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion(higherVersion, Collections.emptySet(), payloadsTruncated);

int mapSize = mapSinceVersion.size();
assertThat(mapSize, is(100));

for (Map.Entry<P2PDataStorage.ByteArray, PersistableNetworkPayload> entry : mapSinceVersion.entrySet()) {
var expected = liveDataMap.get(entry.getKey());
assertThat(entry.getValue(), is(expected));
}

assertThat(payloadsTruncated.get(), is(true));
}

@Test
void testKnownHashes() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 3);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(4, 100);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(5, 200);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
"1.9.0", secondVersionStore);

storageService.setStoresByVersion(storeByVersion);

P2PDataStorage.ByteArray excludeFromLiveMap = liveDataMap.keySet().stream()
.findAny()
.orElseThrow();

P2PDataStorage.ByteArray excludeFromSecondStore = secondVersionStore.getMap()
.keySet().stream()
.findAny()
.orElseThrow();

Set<P2PDataStorage.ByteArray> knownHashes = Set.of(excludeFromLiveMap, excludeFromSecondStore);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.7.0", knownHashes, payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = new HashMap<>(liveDataMap);
expected.putAll(firstVersionStore.getMap());
expected.putAll(secondVersionStore.getMap());

expected.remove(excludeFromLiveMap);
expected.remove(excludeFromSecondStore);

assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}

@Test
void testExcludeAllHashes() {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> liveDataMap = storageService.getStore().getMap();
DummyAccountAgeWitnessFactory.addNewAccountAgeWitnessesToMap(liveDataMap, 3);

AccountAgeWitnessStore firstVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(4, 100);
AccountAgeWitnessStore secondVersionStore = DummyAccountAgeWitnessFactory.
createAccountAgeWitnessStore(5, 200);

Map<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storeByVersion = Map.of(
"1.8.0", firstVersionStore,
"1.9.0", secondVersionStore);

storageService.setStoresByVersion(storeByVersion);

Set<P2PDataStorage.ByteArray> knownHashes = new HashSet<>(liveDataMap.keySet());
Set<P2PDataStorage.ByteArray> secondStoreKeys = secondVersionStore.getMap().keySet();
knownHashes.addAll(secondStoreKeys);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> mapSinceVersion =
storageService.getMapSinceVersion("1.8.5", knownHashes, payloadsTruncated);

Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> expected = Collections.emptyMap();
assertThat(mapSinceVersion, is(expected));
assertThat(payloadsTruncated.get(), is(false));
}
}
10 changes: 7 additions & 3 deletions p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
Expand Down Expand Up @@ -73,7 +74,7 @@ class RequestDataHandler implements MessageListener {
///////////////////////////////////////////////////////////////////////////////////////////

public interface Listener {
void onComplete(boolean wasTruncated);
void onComplete(boolean wasTruncated, String responderVersion);

@SuppressWarnings("UnusedParameters")
void onFault(String errorMessage, @SuppressWarnings("SameParameterValue") @Nullable Connection connection);
Expand All @@ -87,6 +88,7 @@ public interface Listener {
private final NetworkNode networkNode;
private final P2PDataStorage dataStorage;
private final PeerManager peerManager;
private final Optional<String> responderVersion;
private final Listener listener;
private Timer timeoutTimer;
private final int nonce = new Random().nextInt();
Expand All @@ -100,10 +102,12 @@ public interface Listener {
RequestDataHandler(NetworkNode networkNode,
P2PDataStorage dataStorage,
PeerManager peerManager,
Optional<String> responderVersion,
Listener listener) {
this.networkNode = networkNode;
this.dataStorage = dataStorage;
this.peerManager = peerManager;
this.responderVersion = responderVersion;
this.listener = listener;
}

Expand All @@ -124,7 +128,7 @@ void requestData(NodeAddress nodeAddress, boolean isPreliminaryDataRequest) {
if (isPreliminaryDataRequest)
getDataRequest = dataStorage.buildPreliminaryGetDataRequest(nonce);
else
getDataRequest = dataStorage.buildGetUpdatedDataRequest(networkNode.getNodeAddress(), nonce);
getDataRequest = dataStorage.buildGetUpdatedDataRequest(networkNode.getNodeAddress(), nonce, responderVersion);

if (timeoutTimer == null) {
timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions
Expand Down Expand Up @@ -201,7 +205,7 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) {
connection.getPeersNodeAddressOptional().get());

cleanup();
listener.onComplete(getDataResponse.isWasTruncated());
listener.onComplete(getDataResponse.isWasTruncated(), getDataResponse.getVersion());
} else {
log.warn("Nonce not matching. That can happen rarely if we get a response after a canceled " +
"handshake (timeout causes connection close but peer might have sent a msg before " +
Expand Down
Loading
Loading