Merge pull request #15 from ManfredKarrer/sync-with-voting

Sync up with changes from voting branch

src/main/java/bisq/network/p2p/P2PModule.java

@@ -27,6 +27,9 @@
 import bisq.network.p2p.peers.keepalive.KeepAliveManager;
 import bisq.network.p2p.peers.peerexchange.PeerExchangeManager;
 import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadListService;
+import bisq.network.p2p.storage.persistence.ProtectedDataStoreService;
 
 import bisq.common.app.AppModule;
 
@@ -51,6 +54,9 @@ protected void configure() {
         bind(P2PService.class).in(Singleton.class);
         bind(PeerManager.class).in(Singleton.class);
         bind(P2PDataStorage.class).in(Singleton.class);
+        bind(AppendOnlyDataStoreService.class).in(Singleton.class);
+        bind(ProtectedDataStoreService.class).in(Singleton.class);
+        bind(PersistableNetworkPayloadListService.class).in(Singleton.class);
         bind(RequestDataManager.class).in(Singleton.class);
         bind(PeerExchangeManager.class).in(Singleton.class);
         bind(KeepAliveManager.class).in(Singleton.class);

src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java

@@ -140,7 +140,7 @@ private Set<PersistableNetworkPayload> getFilteredPersistableNetworkPayload(GetD
         final Set<P2PDataStorage.ByteArray> tempLookupSet = new HashSet<>();
         Set<P2PDataStorage.ByteArray> excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys());
 
-        return dataStorage.getPersistableNetworkPayloadList().getMap().entrySet().stream()
+        return dataStorage.getAppendOnlyDataStoreMap().entrySet().stream()
                 .filter(e -> !excludedKeysAsByteArray.contains(e.getKey()))
                 .map(Map.Entry::getValue)
                 .filter(payload -> (connection.noCapabilityRequiredOrCapabilityIsSupported(getDataRequest)))

src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java

@@ -124,13 +124,13 @@ public void requestData(NodeAddress nodeAddress, boolean isPreliminaryDataReques
             // PersistedStoragePayload items don't get removed, so we don't have an issue with the case that
             // an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would
             // miss that event if we do not load the full set or use some delta handling.
-            Set<byte[]> excludedKeys = dataStorage.getPersistableNetworkPayloadList().getMap().entrySet().stream()
-                    .map(e -> e.getKey().bytes)
+            Set<byte[]> excludedKeys = dataStorage.getAppendOnlyDataStoreMap().keySet().stream()
+                    .map(e -> e.bytes)
                     .collect(Collectors.toSet());
 
-            Set<byte[]> excludedKeysFromPersistedEntryMap = dataStorage.getPersistedEntryMap().getMap().entrySet()
+            Set<byte[]> excludedKeysFromPersistedEntryMap = dataStorage.getProtectedDataStoreMap().keySet()
                     .stream()
-                    .map(e -> e.getKey().bytes)
+                    .map(e -> e.bytes)
                     .collect(Collectors.toSet());
 
             excludedKeys.addAll(excludedKeysFromPersistedEntryMap);

src/main/java/bisq/network/p2p/storage/SequenceNumberMap.java

@@ -15,7 +15,9 @@
  * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
  */
 
-package bisq.network.p2p.storage;
+package bisq.network.p2p.storage.persistence;
+
+import bisq.network.p2p.storage.P2PDataStorage;
 
 import bisq.common.proto.persistable.PersistableEnvelope;
 

src/main/java/bisq/network/p2p/storage/payload/ProtectedStorageEntry.java

@@ -88,6 +88,11 @@ public Message toProtoMessage() {
                 .build();
     }
 
+    public PB.ProtectedStorageEntry toProtectedStorageEntry() {
+        return (PB.ProtectedStorageEntry) toProtoMessage();
+
+    }
+
     public static ProtectedStorageEntry fromProto(PB.ProtectedStorageEntry proto,
                                                   NetworkProtoResolver resolver) {
         return new ProtectedStorageEntry(proto.getCreationTimeStamp(),

src/main/java/bisq/network/p2p/storage/PersistableNetworkPayloadMapListener.java → src/main/java/bisq/network/p2p/storage/persistence/AppendOnlyDataStoreListener.java

@@ -15,10 +15,10 @@
  * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
  */
 
-package bisq.network.p2p.storage;
+package bisq.network.p2p.storage.persistence;
 
 import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
 
-public interface PersistableNetworkPayloadMapListener {
+public interface AppendOnlyDataStoreListener {
     void onAdded(PersistableNetworkPayload payload);
 }

src/main/java/bisq/network/p2p/storage/persistence/AppendOnlyDataStoreService.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.network.p2p.storage.persistence;
+
+import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
+
+import bisq.common.proto.persistable.PersistableEnvelope;
+
+import javax.inject.Inject;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class AppendOnlyDataStoreService {
+    private List<StoreService<? extends PersistableEnvelope, PersistableNetworkPayload>> services = new ArrayList<>();
+
+    // We do not add PersistableNetworkPayloadListService to the services list as it it deprecated and used only to
+    // transfer old persisted data to the new data structure.
+    private PersistableNetworkPayloadListService persistableNetworkPayloadListService;
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // Constructor
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    @Inject
+    public AppendOnlyDataStoreService(PersistableNetworkPayloadListService persistableNetworkPayloadListService) {
+        this.persistableNetworkPayloadListService = persistableNetworkPayloadListService;
+    }
+
+    public void addService(StoreService<? extends PersistableEnvelope, PersistableNetworkPayload> service) {
+        services.add(service);
+    }
+
+    public void readFromResources(String postFix) {
+        services.forEach(service -> service.readFromResources(postFix));
+
+        transferDeprecatedDataStructure();
+    }
+
+    private void transferDeprecatedDataStructure() {
+        // We read the file if it exists in the db folder
+        persistableNetworkPayloadListService.readStore();
+        // Transfer the content to the new services
+        persistableNetworkPayloadListService.getMap().forEach(this::put);
+        // We are done with the transfer, now let's remove the file
+        persistableNetworkPayloadListService.removeFile();
+    }
+
+    public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMap() {
+        return services.stream()
+                .flatMap(service -> service.getMap().entrySet().stream())
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
+    public void put(P2PDataStorage.ByteArray hashAsByteArray, PersistableNetworkPayload payload) {
+        services.stream()
+                .filter(service -> service.canHandle(payload))
+                .forEach(service -> {
+                    service.putIfAbsent(hashAsByteArray, payload);
+                });
+    }
+}

src/main/java/bisq/network/p2p/storage/PersistableNetworkPayloadList.java → src/main/java/bisq/network/p2p/storage/persistence/PersistableNetworkPayloadList.java

@@ -15,8 +15,9 @@
  * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
  */
 
-package bisq.network.p2p.storage;
+package bisq.network.p2p.storage.persistence;
 
+import bisq.network.p2p.storage.P2PDataStorage;
 import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
 
 import bisq.common.proto.persistable.PersistableEnvelope;
@@ -41,20 +42,21 @@
 // The class name should be map not list but we want to stick with the PB definition name and that cannot be changed
 // without breaking backward compatibility.
 // TODO at next hard fork we can rename the PB definition and class name.
+@Deprecated
 @Slf4j
 public class PersistableNetworkPayloadList implements PersistableEnvelope {
     @Getter
     private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
 
-    PersistableNetworkPayloadList() {
+    public PersistableNetworkPayloadList() {
     }
 
 
     ///////////////////////////////////////////////////////////////////////////////////////////
     // PROTO BUFFER
     ///////////////////////////////////////////////////////////////////////////////////////////
 
-    private PersistableNetworkPayloadList(Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map) {
+    public PersistableNetworkPayloadList(Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map) {
         this.map.putAll(map);
     }
 
@@ -79,4 +81,8 @@ public static PersistableEnvelope fromProto(PB.PersistableNetworkPayloadList pro
                 });
         return new PersistableNetworkPayloadList(map);
     }
+
+    public boolean containsKey(P2PDataStorage.ByteArray hash) {
+        return map.containsKey(hash);
+    }
 }