[INDY-1057] Get rid of RAET code (hyperledger#512)

* Getting rid of raet stuff Signed-off-by: ArtObr <[email protected]> * Additional remove of raet. Remove raet from comments and README Signed-off-by: ArtObr <[email protected]> * Flake8 errors fixed Signed-off-by: ArtObr <[email protected]> * Resolve dependecy issues Signed-off-by: ArtObr <[email protected]> * Dependency fixes, return scripts. Signed-off-by: ArtObr <[email protected]> * Little fixes Signed-off-by: ArtObr <[email protected]>
wiredhikari · Feb 20, 2018 · 794e8e5 · 794e8e5
1 parent 9fbb725
commit 794e8e5
Show file tree

Hide file tree

Showing 41 changed files with 32 additions and 1,751 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

diff --git a/Jenkinsfile.cd b/Jenkinsfile.cd
@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

diff --git a/Jenkinsfile.ci b/Jenkinsfile.ci
@@ -128,7 +128,6 @@ def staticCodeValidation() {
 
 def tests = [
     stp: { python ->
-        test(testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml", python: python)
         test(testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml", python: python)
     },
     ledger: { python ->

diff --git a/README.md b/README.md
@@ -49,8 +49,7 @@ mechanism that Plenum needs.
     - state storage using python 3 version of Ethereum's Patricia Trie
 - stp:
     - secure transport abstraction
-    - it has two implementations: RAET and ZeroMQ
-    - Although RAET implementation is there, it's not supported anymore, and [ZeroMQ](http://zeromq.org/) is the default secure transport in plenum. 
+    - it has [ZeroMQ](http://zeromq.org/) implementations
 - storage:
     - key-value storage abstractions
     - contains [leveldb](http://leveldb.org/) implementation as the main key-valued storage used in Plenum (for ledger, state, etc.)

diff --git a/build-scripts/ubuntu-1604/build-3rd-parties.sh b/build-scripts/ubuntu-1604/build-3rd-parties.sh
@@ -44,7 +44,8 @@ build_from_pypi base58 0.2.4
 build_from_pypi prompt-toolkit 0.57
 build_from_pypi rlp 0.5.1
 build_from_pypi sha3 0.2.1
-build_from_pypi raet 0.6.6
+build_from_pypi libnacl 1.6.1
+build_from_pypi six 1.11.0
 build_from_pypi pyzmq 16.0.2
 build_from_pypi intervaltree 2.1.0
 build_from_pypi portalocker 0.5.7

diff --git a/plenum/cli/cli.py b/plenum/cli/cli.py
@@ -40,7 +40,6 @@
 from stp_core.crypto.util import cleanSeed, seedFromHex
 from stp_core.network.port_dispenser import genHa
 from stp_core.types import HA
-from stp_raet.util import getLocalEstateData
 from plenum.common.config_helper import PNodeConfigHelper
 
 import configparser
@@ -231,20 +230,13 @@ def __init__(self, looper, basedirpath: str, ledger_base_dir: str, nodeReg=None,
             eventloop=eventloop,
             output=out)
 
-        # RAETVerbosity = getRAETLogLevelFromConfig("RAETLogLevelCli",
-        #                                           Console.Wordage.mute,
-        #                                           self.config)
-        # RAETLogFile = getRAETLogFilePath("RAETLogFilePathCli", self.config)
         # Patch stdout in something that will always print *above* the prompt
         # when something is written to stdout.
         sys.stdout = self.cli.stdout_proxy()
 
         if logFileName:
             Logger().enableFileLogging(logFileName)
 
-        # TODO: If we want RAET logging in CLI we need fix this. See INDY-315.
-        # Logger().setupRaet(RAETVerbosity, RAETLogFile)
-
         self.logger = getlogger("cli")
         self.print("\n{}-CLI (c) 2017 Evernym, Inc.".format(self.properName))
         self._actions = []
@@ -1062,8 +1054,7 @@ def newClient(self, clientName,
             if not areKeysSetup(clientName, self.basedirpath):
                 client_addr = genHa(ip='0.0.0.0')
             else:
-                client_addr = tuple(getLocalEstateData(clientName,
-                                                       self.basedirpath)['ha'])
+                raise Exception("Usage of deprecated code")
             nodeReg = None if self.nodeRegLoadedFromFile else self.cliNodeReg
             client = self.ClientClass(clientName,
                                       ha=client_addr,

diff --git a/plenum/client/client.py b/plenum/client/client.py
@@ -107,7 +107,6 @@ def __init__(self,
         self.keys_dir = os.path.join(_keys_dir, "keys")
 
         cha = None
-        # If client information already exists is RAET then use that
         if self.exists(self.stackName, self.keys_dir):
             cha = self.nodeStackClass.getHaFromLocal(
                 self.stackName, self.keys_dir)

diff --git a/plenum/common/ledger_manager.py b/plenum/common/ledger_manager.py
@@ -636,7 +636,7 @@ def _getCatchupReplyForSeqNo(self, ledgerId, seqNo):
         for k, catchupReps in ledgerInfo.recvdCatchupRepliesFrm.items():
             for rep in catchupReps:
                 txns = getattr(rep, f.TXNS.nm)
-                # Transfers of odcits in RAET converts integer keys to string
+
                 if str(seqNo) in txns:
                     return k, rep
 

diff --git a/plenum/common/script_helper.py b/plenum/common/script_helper.py
@@ -14,7 +14,6 @@
 from stp_core.loop.eventually import eventually
 from stp_core.network.port_dispenser import genHa
 from stp_core.types import HA
-from stp_raet.util import getLocalVerKey, getLocalPubKey
 
 NodeInfoFile = "node-info"
 GenTxnFile = "genesis_txn"
@@ -107,10 +106,6 @@ def storeGenTxns(baseDir, txn):
                 isLineNoKey=True)
 
 
-def getStewardKeyFromName(baseDir, name):
-    return getLocalVerKey(name, baseDir)
-
-
 def getAddGenesisHAs(nodeip, nodeport, clientip, clientport):
     vnodeip = nodeip if nodeip else "127.0.0.1"
     vnodeport = nodeport if nodeport else "9701"
@@ -182,41 +177,6 @@ def printGenTxn(txn, displayTxn):
         print('\n' + txn)
 
 
-def getVerKeyFromName(baseDir, roleName):
-    return getLocalVerKey(roleName, baseDir)
-
-
-def getPubKeyFromName(baseDir, roleName):
-    return getLocalPubKey(roleName, baseDir)
-
-
-def exportNodeGenTxn(baseDir, displayTxn, name):
-    nodeInfo = getNodeInfo(baseDir, name)
-    nodeVerKey = getVerKeyFromName(baseDir, name)
-    stewardKey = nodeInfo.get('steward')
-    nodeAddr = nodeInfo.get('nodeAddr')
-    clientAddr = nodeInfo.get('clientAddr')
-
-    txn = 'add genesis transaction {node} with data {"'.format(node=PlenumTransactions.NODE.name) + name + '": {' \
-        '"verkey":' \
-        ' "' + \
-          nodeVerKey + \
-          '", "node_address": "' + nodeAddr + \
-          '", "client_address": "' + clientAddr + '"}, "by":"' + stewardKey + \
-          '"}'
-    storeExportedTxns(baseDir, txn)
-    printGenTxn(txn, displayTxn)
-
-
-def exportStewardGenTxn(baseDir, displayTxn, name):
-    verkey = getLocalVerKey(name, baseDir)
-    txn = 'add genesis transaction {nym} with data  {"'.format(nym=PlenumTransactions.NYM.name) + name + '": {' \
-        '"verkey": "' + verkey + '"} role={role}'.format(
-        role=Roles.STEWARD.name)
-    storeExportedTxns(baseDir, txn)
-    printGenTxn(txn, displayTxn)
-
-
 def submitNodeIpChange(client, stewardWallet, name: str, nym: str,
                        nodeStackHa: HA, clientStackHa: HA):
     (nodeIp, nodePort), (clientIp, clientPort) = nodeStackHa, clientStackHa

diff --git a/plenum/common/stacks.py b/plenum/common/stacks.py
@@ -1,11 +1,9 @@
 from typing import Callable, Any, List, Dict
 
-from plenum import config
 from plenum.common.batched import Batched, logger
 from plenum.common.config_util import getConfig
 from plenum.common.message_processor import MessageProcessor
 from stp_core.common.constants import CONNECTION_PREFIX
-from stp_raet.rstack import SimpleRStack, KITRStack
 from stp_core.types import HA
 from stp_zmq.kit_zstack import KITZStack
 from stp_zmq.simple_zstack import SimpleZStack
@@ -41,8 +39,6 @@ def transmitToClient(self, msg: Any, remoteName: str):
         :param msg: a message
         :param remoteName: the name of the remote
         """
-        # At this time, nodes are not signing messages to clients, beyond what
-        # happens inherently with RAET
         payload = self.prepForSending(msg)
         try:
             if isinstance(remoteName, str):
@@ -84,67 +80,5 @@ def start(self, restricted=None, reSetupAuth=True):
                     extra={"tags": ["node-listening"]})
 
 
-class ClientRStack(SimpleRStack, MessageProcessor):
-    def __init__(self, stackParams: dict, msgHandler: Callable, seed=None):
-        # The client stack needs to be mutable unless we explicitly decide
-        # not to
-        stackParams["mutable"] = stackParams.get("mutable", True)
-        stackParams["messageTimeout"] = config.RAETMessageTimeout
-        SimpleRStack.__init__(self, stackParams, msgHandler)
-        MessageProcessor.__init__(self, allowDictOnly=True)
-        self.connectedClients = set()
-
-    def serviceClientStack(self):
-        newClients = self.connecteds - self.connectedClients
-        self.connectedClients = self.connecteds
-        return newClients
-
-    def newClientsConnected(self, newClients):
-        raise NotImplementedError("{} must implement this method".format(self))
-
-    def transmitToClient(self, msg: Any, remoteName: str):
-        """
-        Transmit the specified message to the remote client specified by `remoteName`.
-
-        :param msg: a message
-        :param remoteName: the name of the remote
-        """
-        # At this time, nodes are not signing messages to clients, beyond what
-        # happens inherently with RAET
-        payload = self.prepForSending(msg)
-        try:
-            self.send(payload, remoteName)
-        except Exception as ex:
-            # TODO: This should not be an error since the client might not have
-            # sent the request to all nodes but only some nodes and other
-            # nodes might have got this request through PROPAGATE and thus
-            # might not have connection with the client.
-            logger.error(
-                "{}{} unable to send message {} to client {}; Exception: {}" .format(
-                    CONNECTION_PREFIX, self, msg, remoteName, ex.__repr__()))
-
-    def transmitToClients(self, msg: Any, remoteNames: List[str]):
-        for nm in remoteNames:
-            self.transmitToClient(msg, nm)
-
-
-class NodeRStack(Batched, KITRStack):
-    def __init__(self, stackParams: dict, msgHandler: Callable,
-                 registry: Dict[str, HA], seed=None, sighex: str=None):
-        Batched.__init__(self)
-        # TODO: Just to get around the restriction of port numbers changed on
-        # Azure. Remove this soon to relax port numbers only but not IP.
-        stackParams["mutable"] = stackParams.get("mutable", True)
-        stackParams["messageTimeout"] = config.RAETMessageTimeout
-        KITRStack.__init__(self, stackParams, msgHandler, registry, sighex)
-        MessageProcessor.__init__(self, allowDictOnly=True)
-
-    def start(self):
-        KITRStack.start(self)
-        logger.info("{}{} listening for other nodes at {}:{}".
-                    format(CONNECTION_PREFIX, self, *self.ha),
-                    extra={"tags": ["node-listening"]})
-
-
-nodeStackClass = NodeZStack if config.UseZStack else NodeRStack
-clientStackClass = ClientZStack if config.UseZStack else ClientRStack
+nodeStackClass = NodeZStack
+clientStackClass = ClientZStack
diff --git a/plenum/config.py b/plenum/config.py
@@ -123,12 +123,6 @@
 DUMP_VALIDATOR_INFO_INIT_SEC = 3
 DUMP_VALIDATOR_INFO_PERIOD_SEC = 60
 
-RAETLogLevel = "terse"
-RAETLogLevelCli = "mute"
-RAETLogFilePath = os.path.expanduser('~/.plenum/raet.log')
-RAETLogFilePathCli = None
-RAETMessageTimeout = 60
-
 # Controls sending of view change messages, a node will only send view change
 # messages if it did not send any sent instance change messages in last
 # `ViewChangeWindowSize` seconds
@@ -177,9 +171,6 @@
 
 log_override_tags = dict(cli={}, demo={})
 
-# TODO needs to be refactored to use a transport protocol abstraction
-UseZStack = True
-
 
 # Number of messages zstack accepts at once
 LISTENER_MESSAGE_QUOTA = 100

diff --git a/plenum/server/node.py b/plenum/server/node.py
@@ -302,10 +302,9 @@ def __init__(self,
 
         # BE CAREFUL HERE
         # This controls which message types are excluded from signature
-        # verification. These are still subject to RAET's signature verification
-        # but client signatures will not be checked on these. Expressly
-        # prohibited from being in this is ClientRequest and Propagation,
-        # which both require client signature verification
+        # verification. Expressly prohibited from being in this is
+        # ClientRequest and Propagation, which both require client
+        # signature verification
         self.authnWhitelist = (
             Nomination,
             Primary,
@@ -2426,7 +2425,7 @@ def verifySignature(self, msg):
         :return: None; raises an exception if the signature is not valid
         """
         if isinstance(msg, self.authnWhitelist):
-            return  # whitelisted message types rely on RAET for authn
+            return
         if isinstance(msg, Propagate):
             typ = 'propagate'
             req = msg.request

diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py
@@ -71,7 +71,7 @@ def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status):
             logger.debug("Remove all the nodes")
             for n in nodeNames:
                 looper.removeProdable(nodeSet.nodes[n])
-                nodeSet.removeNode(n, shouldClean=False)
+                nodeSet.removeNode(n)
 
             looper.runFor(10)
 

diff --git a/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py b/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py
@@ -29,7 +29,7 @@
 
 # TODO: This test passes but it is observed that PREPAREs are not received at
 # newly added node. If the stop and start steps are omitted then PREPAREs are
-# received. Conclusion is that due to node restart, RAET is losing messages
+# received. Conclusion is that due to node restart, ZMQ is losing messages
 # but its weird since prepares and commits are received which are sent before
 # and after prepares, respectively. Here is the pivotal link
 # https://www.pivotaltracker.com/story/show/127897273

diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py
@@ -271,11 +271,6 @@ def updateNodeData(looper, stewardClient, stewardWallet, node, node_data):
     req = sendUpdateNode(stewardClient, stewardWallet, node, node_data)
     waitForSufficientRepliesForRequests(looper, stewardClient,
                                         requests=[req])
-    # TODO: Not needed in ZStack, remove once raet is removed
-    node.nodestack.clearLocalKeep()
-    node.nodestack.clearRemoteKeeps()
-    node.clientstack.clearLocalKeep()
-    node.clientstack.clearRemoteKeeps()
 
 
 def sdk_send_update_node(looper, sdk_pool, sdk_wallet_steward, node, node_data):

diff --git a/plenum/test/pool_transactions/test_client_change_ha.py b/plenum/test/pool_transactions/test_client_change_ha.py
@@ -28,10 +28,6 @@ def testClientReconnectUsingDifferentHa(looper, txnPoolNodeSet, tdirWithPoolTxns
     client.stop()
     looper.removeProdable(client)
 
-    # Removing RAET keep directory otherwise the client will use the same port
-    #  since it will a directory of its name in the keep
-    shutil.rmtree(keys_dir, ignore_errors=True)
-
     ha = genHa()
     client, _ = genTestClient(txnPoolNodeSet, identifier=wallet.defaultId,
                               ha=ha, tmpdir=tdirWithClientPoolTxns,

diff --git a/plenum/test/test_node.py b/plenum/test/test_node.py
@@ -541,11 +541,8 @@ def addNode(self, name: str) -> TestNode:
         self.__dict__[name] = node
         return node
 
-    def removeNode(self, name, shouldClean):
+    def removeNode(self, name):
         self.nodes[name].stop()
-        if shouldClean:
-            self.nodes[name].nodestack.keep.clearAllDir()
-            self.nodes[name].clientstack.keep.clearAllDir()
         del self.nodes[name]
         del self.__dict__[name]
         # del self.nodeRegistry[name]
@@ -906,7 +903,7 @@ def prepareNodeSet(looper: Looper, nodeSet: TestNodeSet):
     # Remove all the nodes
     for n in list(nodeSet.nodes.keys()):
         looper.removeProdable(nodeSet.nodes[n])
-        nodeSet.removeNode(n, shouldClean=False)
+        nodeSet.removeNode(n)
 
 
 def checkViewChangeInitiatedForNode(node: TestNode, proposedViewNo: int):

diff --git a/plenum/test/test_node_connection.py b/plenum/test/test_node_connection.py
@@ -248,10 +248,5 @@ def testNodeRemoveUnknownRemote(allPluginsPath, tdir_for_func, tconf_for_func,
 
     stopNodes([C, ], looper)
 
-    def chk():
-        assert C.name not in B.nodestack.nameRemotes
-        assert C.name not in A.nodestack.nameRemotes
-
     timeout = waits.expectedPoolInterconnectionTime(len(nodeReg))
-    looper.run(eventually(chk, retryWait=2, timeout=timeout))
     stopNodes([A, B], looper)