Merge pull request #1725 from ergoplatform/v4.0.32

Candidate for release 4.0.32

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.31`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.32`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala

@@ -4,7 +4,7 @@ package scorex.crypto.authds.avltree.batch
 import scorex.crypto.authds.{ADKey, Balance}
 import scorex.crypto.hash.{CryptographicHash, Digest}
 import scorex.db.LDBVersionedStore
-
+import InternalNode.InternalNodePrefix
 
 class ProxyInternalProverNode[D <: Digest](protected var pk: ADKey,
                                            val lkey: ADKey,
@@ -15,6 +15,10 @@ class ProxyInternalProverNode[D <: Digest](protected var pk: ADKey,
                                            nodeParameters: NodeParameters)
   extends InternalProverNode(k = pk, l = null, r = null, b = pb)(phf) {
 
+  override protected def computeLabel: D = {
+    hf.hash(Array(InternalNodePrefix, b), lkey, rkey)
+  }
+
   override def left: ProverNodes[D] = {
     if (l == null) l = VersionedLDBAVLStorage.fetch[D](lkey)
     l

src/main/resources/api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "4.0.31"
+  version: "4.0.32"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:

src/main/resources/application.conf

@@ -388,7 +388,7 @@ scorex {
     nodeName = "ergo-node"
 
     # Network protocol version to be sent in handshakes
-    appVersion = 4.0.31
+    appVersion = 4.0.32
 
     # Network agent name. May contain information about client code
     # stack, starting from core code-base up to the end graphical interface.

src/main/resources/mainnet.conf

@@ -49,8 +49,8 @@ ergo {
     # The node still applying transactions to UTXO set and so checks UTXO set digests for each block.
     # Block at checkpoint height is to be checked against expected one.
     checkpoint = {
-      height = 755464
-      blockId = "48542c37b030e7d509754434aa97cf9691ad0ada30d60711cd345862a168e0fb"
+      height = 777670
+      blockId = "157ecf59b067fdc81921432d7b8c4cb30890f18056788fdc32e433756a9fbb78"
     }
 
     # List with hex-encoded identifiers of transactions banned from getting into memory pool
@@ -65,7 +65,7 @@ scorex {
   network {
     magicBytes = [1, 0, 2, 4]
     bindAddress = "0.0.0.0:9030"
-    nodeName = "ergo-mainnet-4.0.31"
+    nodeName = "ergo-mainnet-4.0.32"
     nodeName = ${?NODENAME}
     knownPeers = [
       "213.239.193.208:9030",

src/main/resources/testnet.conf

@@ -77,7 +77,7 @@ scorex {
   network {
     magicBytes = [2, 0, 0, 2]
     bindAddress = "0.0.0.0:9020"
-    nodeName = "ergo-testnet-4.0.31"
+    nodeName = "ergo-testnet-4.0.32"
     nodeName = ${?NODENAME}
     knownPeers = [
       "213.239.193.208:9020",

src/main/scala/org/ergoplatform/local/MempoolAuditor.scala

@@ -35,7 +35,7 @@ class MempoolAuditor(nodeViewHolderRef: ActorRef,
   }
 
   override def postStop(): Unit = {
-    logger.info("Mempool auditor stopped")
+    log.info("Mempool auditor stopped")
     super.postStop()
   }
 

src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala

@@ -209,14 +209,16 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
 
   /**
     * Helper method to validate reemission rules according to EIP-27
+    *
+    * @param boxesToSpend     - inputs of transaction
+    * @param outputCandidates - outputs of the transaction
+    * @param stateContext     - validation context
     */
-  def verifyReemissionSpending(boxesToSpend: IndexedSeq[ErgoBox],
-                               outputCandidates: Seq[ErgoBoxCandidate],
-                               stateContext: ErgoStateContext): Try[Unit] = {
-    Try {
-      // we check that we're in utxo mode, as eip27Supported flag available only in this mode
-      // if we're in digest mode, skip validation
-      // todo: this check could be removed after EIP-27 activation
+  private def verifyReemissionSpending(boxesToSpend: IndexedSeq[ErgoBox],
+                                       outputCandidates: Seq[ErgoBoxCandidate],
+                                       stateContext: ErgoStateContext): Try[Unit] = {
+    val res: Try[Unit] = Try {
+
       lazy val reemissionSettings = stateContext.ergoSettings.chainSettings.reemission
       lazy val reemissionRules = reemissionSettings.reemissionRules
 
@@ -233,18 +235,23 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
 
       val activationHeight = reemissionSettings.activationHeight
 
-      if (stateContext.currentHeight >= activationHeight) {
+      if (height >= activationHeight) { // we check EIP-27 rules only since activation height
         // reemission check logic below
-        var reemissionSpending = false
+        var reemissionSpending = false // flag indicating that inputs have re-emission tokens
         boxesToSpend.foreach { box =>
           // checking EIP-27 rules for emission box
           // for efficiency, skip boxes with less than 100K ERG
+          // secure, as charging emission box will be stopped
+          // before 100K ERG left in the emission box
+          // todo: for efficiency, we can raise the bar probably
           if (box.value > 100000 * EmissionRules.CoinsInOneErgo) {
             // on activation height, emissionNft is not in emission box yet, but in injection box
             // injection box index (1) is enforced by injection box contract
             if (box.tokens.contains(emissionNftId) ||
               (height == activationHeight && boxesToSpend(1).tokens.contains(emissionNftId))) {
 
+              // in this branch, we are checking spending of re-emission tokens from the emission boxes
+
               // if emission contract NFT is in the input, remission tokens should be there also
               val reemissionTokensIn = if (height == activationHeight) {
                 boxesToSpend(1).tokens.getOrElse(reemissionTokenId, 0L)
@@ -272,12 +279,14 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
               //we're checking how emission box is paying reemission tokens below
               val emissionTokensOut = emissionOut.tokens.getOrElse(reemissionTokenId, 0L)
               val rewardsTokensOut = rewardsOut.tokens.getOrElse(reemissionTokenId, 0L)
+              // it is prohibited to burn re-emission tokens on spending the emission box
               require(reemissionTokensIn == emissionTokensOut + rewardsTokensOut, "Reemission tokens not preserved")
 
               val properReemissionRewardPart = reemissionRules.reemissionForHeight(height, emissionRules)
               require(rewardsTokensOut == properReemissionRewardPart, "Rewards out condition violated")
             } else {
-              //this path can be removed after EIP-27 activation
+              // this path can be removed after EIP-27 activation in 5.0
+              // that is not so easy though, see https://github.com/ergoplatform/ergo/issues/1736
               if (height >= activationHeight && box.ergoTree == chainSettings.monetary.emissionBoxProposition) {
                 //we require emission contract NFT and reemission token to be presented in emission output
                 val emissionOutTokens = outputCandidates(0).tokens
@@ -300,7 +309,14 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
           log.debug(s"Reemission tokens to burn: $toBurn")
           val reemissionOutputs = outputCandidates.filter { out =>
             require(!out.tokens.contains(reemissionTokenId), "outputs contain reemission token")
-            out.ergoTree == payToReemissionContract
+            // we compare by trees in the mainnet (to avoid disagreement with versions 4.0.29-31 doing that),
+            // and by propositions in the testnet (as there are v0 & v1 transactions paying to pay-to-reemission there)
+            // see https://github.com/ergoplatform/ergo/pull/1728 for details.
+            if (chainSettings.isMainnet) {
+              out.ergoTree == payToReemissionContract
+            } else {
+              out.ergoTree.toProposition(true) == payToReemissionContract.toProposition(true)
+            }
           }
           val sentToReemission = reemissionOutputs.map(_.value).sum
           require(sentToReemission == toBurn, "Burning condition violated")
@@ -309,6 +325,13 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
         Success(())
       }
     }
+
+    res match {
+      case Failure(e) => log.error(s"EIP-27 check failed due to ${e.getMessage} : ", e)
+      case _ =>
+    }
+
+    res
   }
 
   /**

src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala

@@ -17,7 +17,6 @@ import org.ergoplatform.nodeView.mempool.{ErgoMemPool, ErgoMemPoolReader}
 import org.ergoplatform.settings.{Constants, ErgoSettings}
 import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{ChainIsHealthy, ChainIsStuck, GetNodeViewChanges, IsChainHealthy, ModifiersFromRemote, TransactionsFromRemote}
 import org.ergoplatform.nodeView.ErgoNodeViewHolder._
-import scorex.core.app.Version
 import scorex.core.consensus.History.{Equal, Fork, Nonsense, Older, Unknown, Younger}
 import scorex.core.consensus.{HistoryReader, SyncInfo}
 import scorex.core.network.ModifiersStatus.Requested
@@ -173,11 +172,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
   /**
     * Whether neighbour peer `remote` supports sync protocol V2.
     */
-  def syncV2Supported(remote: ConnectedPeer): Boolean = {
-    // If neighbour version is >= 4.0.16, the neighbour supports sync V2
-    val syncV2Version = Version(4, 0, 16)
-    remote.peerInfo.exists(_.peerSpec.protocolVersion >= syncV2Version)
-  }
+  def syncV2Supported(remote: ConnectedPeer): Boolean = SyncV2Filter.condition(remote)
 
   /**
     * Send synchronization statuses to neighbour peers
@@ -344,6 +339,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
 
       case Older =>
         log.debug(s"Peer $remote is older, its height ${syncInfo.height}")
+        applyValidContinuationHeaderV2(syncInfo, hr, remote)
 
       case Equal =>
         // does nothing for `Equal`
@@ -356,6 +352,26 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     }
   }
 
+  /**
+    * Calculates new continuation header from syncInfo message if any, validates it and sends it
+    * to nodeViewHolder as a remote modifier for it to be applied
+    * @param syncInfo other's node sync info
+    */
+  private def applyValidContinuationHeaderV2(syncInfo: ErgoSyncInfoV2, history: ErgoHistory, peer: ConnectedPeer): Unit =
+    history.continuationHeaderV2(syncInfo).foreach { continuationHeader =>
+      history.applicableTry(continuationHeader) match {
+        case Failure(e) if e.isInstanceOf[MalformedModifierError] =>
+          log.warn(s"Header from syncInfoV2 ${continuationHeader.encodedId} is invalid", e)
+        case _ =>
+          log.info(s"Applying valid syncInfoV2 header ${continuationHeader.encodedId} and downloading its block sections")
+          viewHolderRef ! ModifiersFromRemote(Seq(continuationHeader))
+          val modifiersToDownload = history.requiredModifiersForHeader(continuationHeader)
+          modifiersToDownload.foreach { case (modifierTypeId, modifierId) =>
+            downloadModifiers(Seq(modifierId), modifierTypeId, peer)
+          }
+      }
+    }
+
   /**
     * Headers should be downloaded from an Older node, it is triggered by received sync message from an older node
     * @param callingPeer that can be used to download headers, it must be Older
@@ -384,23 +400,25 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
       }.map { case (syncState, peers) =>
         val peersFiltered =
           if (settings.nodeSettings.stateType == StateType.Digest) {
-            // 4.0.21.1 allows for downloading ADProofs that are too big in block at 667614
-            val requiredVersion = Version(4, 0, 22)
-            peers.filter { cp =>
-              val version = cp.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial)
-              version.compare(requiredVersion) >= 0
-            }
+            DigestModeFilter.filter(peers)
           } else {
-            // filter out peers of 4.0.17 or 4.0.18 version as they are delivering broken modifiers
-            peers.filterNot { cp =>
-              val version = cp.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial)
-              version == Version.v4017 || version == Version.v4018
-            }
+            BrokenModifiersFilter.filter(peers)
           }
         syncState -> peersFiltered
       }
   }
 
+  /**
+    * Set modifiers of particular type as Requested and download them from given peer and periodically check for delivery
+    */
+  private def downloadModifiers(modifierIds: Seq[ModifierId], modifierTypeId: ModifierTypeId, peer: ConnectedPeer): Unit = {
+    deliveryTracker.setRequested(modifierIds, modifierTypeId, Some(peer)) { deliveryCheck =>
+      context.system.scheduler.scheduleOnce(deliveryTimeout, self, deliveryCheck)
+    }
+    val msg = Message(requestModifierSpec, Right(InvData(modifierTypeId, modifierIds)), None)
+    networkControllerRef ! SendToNetwork(msg, SendToPeer(peer))
+  }
+
   /**
     * Modifier download method that is given min/max constraints for modifiers to download from peers.
     * It sends requests for modifiers to given peers in optimally sized batches.
@@ -427,11 +445,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
         }
         // bucket represents a peer and a modifierType as we cannot send mixed types to a peer
         modifiersByBucket.foreach { case ((peer, modifierTypeId), modifierIds) =>
-          deliveryTracker.setRequested(modifierIds, modifierTypeId, Some(peer)) { deliveryCheck =>
-            context.system.scheduler.scheduleOnce(deliveryTimeout, self, deliveryCheck)
-          }
-          val msg = Message(requestModifierSpec, Right(InvData(modifierTypeId, modifierIds)), None)
-          networkControllerRef ! SendToNetwork(msg, SendToPeer(peer))
+          downloadModifiers(modifierIds, modifierTypeId, peer)
         }
       }
 
@@ -495,6 +509,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
                                             remote: ConnectedPeer): Iterable[M] = {
     modifiers.flatMap { case (id, bytes) =>
       if (typeId == Transaction.ModifierTypeId && bytes.length > settings.nodeSettings.maxTransactionSize) {
+        deliveryTracker.setInvalid(id, typeId)
         penalizeMisbehavingPeer(remote)
         log.warn(s"Transaction size ${bytes.length} from ${remote.toString} exceeds limit ${settings.nodeSettings.maxTransactionSize}")
         None

src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala

@@ -83,8 +83,11 @@ final case class ErgoSyncTracker(system: ActorSystem,
 
   def clearStatus(remote: InetSocketAddress): Unit = {
     statuses.find(_._1.connectionId.remoteAddress == remote) match {
-      case Some((peer, _)) => statuses -= peer
-      case None => log.warn(s"Trying to clear status for $remote, but it is not found")
+      case Some((peer, _)) =>
+        statuses -= peer
+        heights -= peer
+      case None =>
+        log.warn(s"Trying to clear status for $remote, but it is not found")
     }
   }
 
@@ -107,7 +110,13 @@ final case class ErgoSyncTracker(system: ActorSystem,
 
   protected def numOfSeniors(): Int = statuses.count(_._2.status == Older)
 
-  def maxHeight(): Option[Int] = if(heights.nonEmpty) Some(heights.maxBy(_._2)._2) else None
+  def maxHeight(): Option[Int] = {
+    if (heights.nonEmpty) {
+      Some(heights.maxBy(_._2)._2)
+    } else {
+      None
+    }
+  }
 
   /**
     * Return the peers to which this node should send a sync signal, including:

src/main/scala/org/ergoplatform/network/PeerFilteringRule.scala

@@ -0,0 +1,72 @@
+package org.ergoplatform.network
+
+import scorex.core.app.Version
+import scorex.core.network.ConnectedPeer
+
+/**
+  * Basic abstract component describing an action of choosing peers from available ones
+  * based on peer version (and other properties).
+  */
+sealed trait PeerFilteringRule {
+
+  /**
+    * @param version - peer version
+    * @return whether peer of this version should be selected
+    */
+  def condition(version: Version): Boolean
+
+  /**
+    * @param peer - peer
+    * @return - whether the peer should be selected
+    */
+  def condition(peer: ConnectedPeer): Boolean = {
+    val version = peer.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial)
+    condition(version)
+  }
+
+  /**
+    * Select peers satisfying the condition from provided ones
+    * @param peers - unfiltered peers
+    * @return filtered peers
+    */
+  def filter(peers: Iterable[ConnectedPeer]): Iterable[ConnectedPeer] = {
+    peers.filter(cp => condition(cp))
+  }
+
+}
+
+
+/**
+  * 4.0.22+ allow for downloading ADProofs that are too big in block at 667614
+  * for prior versions, a peer will not deliver block # 667614 and some other blocks
+  */
+object DigestModeFilter extends PeerFilteringRule {
+
+  override def condition(version: Version): Boolean = {
+      version.compare(Version.v4022) >= 0
+  }
+
+}
+
+/**
+  * Filter out peers of 4.0.17 or 4.0.18 version as they are delivering broken modifiers
+  */
+object BrokenModifiersFilter extends PeerFilteringRule {
+
+  override def condition(version: Version): Boolean = {
+    version != Version.v4017 && version != Version.v4018
+  }
+
+}
+
+/**
+  * If peer's version is >= 4.0.16, the peer is supporting sync V2
+  */
+object SyncV2Filter extends PeerFilteringRule {
+
+  override def condition(version: Version): Boolean = {
+    val syncV2Version = Version(4, 0, 16)
+    version.compare(syncV2Version) >= 0
+  }
+
+}