From 9fc90e42c7524d08f5748a8af1237de62a5dc6cf Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 11 Oct 2019 01:18:27 +0200 Subject: [PATCH 01/38] add tcp server and client connections (wip) --- aea/connections/oef/connection.yaml | 7 +- aea/connections/tcp_client/__init__.py | 21 ++ aea/connections/tcp_client/connection.py | 208 ++++++++++++++++++++ aea/connections/tcp_client/connection.yaml | 15 ++ aea/connections/tcp_server/__init__.py | 21 ++ aea/connections/tcp_server/connection.py | 214 +++++++++++++++++++++ aea/connections/tcp_server/connection.yaml | 15 ++ 7 files changed, 496 insertions(+), 5 deletions(-) create mode 100644 aea/connections/tcp_client/__init__.py create mode 100644 aea/connections/tcp_client/connection.py create mode 100644 aea/connections/tcp_client/connection.yaml create mode 100644 aea/connections/tcp_server/__init__.py create mode 100644 aea/connections/tcp_server/connection.py create mode 100644 aea/connections/tcp_server/connection.yaml diff --git a/aea/connections/oef/connection.yaml b/aea/connections/oef/connection.yaml index 7b040f3707..0f83c02326 100644 --- a/aea/connections/oef/connection.yaml +++ b/aea/connections/oef/connection.yaml @@ -6,8 +6,5 @@ url: "" class_name: OEFConnection supported_protocols: ["oef"] config: - addr: 127.0.0.1 - port: 10000 -dependencies: - - colorlog - - oef \ No newline at end of file + bind: /tmp/socket +dependencies: [] \ No newline at end of file diff --git a/aea/connections/tcp_client/__init__.py b/aea/connections/tcp_client/__init__.py new file mode 100644 index 0000000000..d6c337b78a --- /dev/null +++ b/aea/connections/tcp_client/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP connection (client side).""" diff --git a/aea/connections/tcp_client/connection.py b/aea/connections/tcp_client/connection.py new file mode 100644 index 0000000000..cba903693a --- /dev/null +++ b/aea/connections/tcp_client/connection.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP server.""" +import asyncio +import logging +import queue +import struct +from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, transports +from ipaddress import IPv4Address +from threading import Thread +from typing import Dict, Optional, Tuple + +from aea.configurations.base import ConnectionConfig +from aea.connections.base import Channel, Connection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + +STUB_DIALOGUE_ID = 0 + + +class TCPClientChannel(Channel): + """Channel implementation for the local node.""" + + def __init__(self, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): + """ + Initialize a TCP channel. + + :param address: the socket bind address. + :param unix: whether it's a unix server or a networked. + :param loop: the asyncio loop. + """ + self.in_queue = queue.Queue() + self.address = address + self.unix = unix + self._loop = asyncio.new_event_loop() if loop is None else loop + self._thread = None # type: Optional[Thread] + + self._reader, self._writer = (None, None) + + async def _recv(self): + data = await self._reader.read(len(struct.pack("I", 0))) + nbytes = struct.unpack("I", data)[0] + nbytes_read = 0 + data = b"" + while nbytes_read < nbytes: + data += (await self._reader.read(nbytes - nbytes_read)) + nbytes_read = len(data) + + return data + + async def _send(self, data: bytes): + nbytes = struct.pack("I", len(data)) + await self._writer.write(nbytes) + await self._writer.write(data) + await self._writer.drain() + + async def _recv_loop(self): + data = await self._recv() + envelope = Envelope.decode(data) + self.in_queue.put_nowait(envelope) + await self._recv_loop() + + def connect(self): + """ + Set up the connection. + + :return: A queue or None. + """ + self._thread = Thread(target=self._loop.run_forever) + self._thread.start() + + if self.unix: + coro = asyncio.open_unix_connection(path=self.address, loop=self._loop) + else: + ip, port = self.address.split(":") + port = int(port) + coro = asyncio.open_connection(ip, port, loop=self._loop, start_serving=False) + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + self._reader, self._writer = future.result() + + + def disconnect(self) -> None: + """ + Tear down the connection. + + :return: None. + """ + self._reader.close() + self._writer.close() + self._loop.stop() + + def send(self, envelope: Envelope) -> None: + """ + Send an envelope. + + :param envelope: the envelope to send. + :return: None. + """ + asyncio.run_coroutine_threadsafe(self._send(envelope.encode()), loop=self._loop) + + def recv(self, block=True, timeout=None) -> Optional[Envelope]: + """Receive an envelope.""" + try: + self.in_queue.get(block=block, timeout=timeout) + except queue.Empty: + return None + + +class TCPClientConnection(Connection): + """Implementation of a TCP server connection.""" + + def __init__(self, address: str): + """ + Initialize a TCP connection + + :param address: the address + """ + super().__init__() + self.address = address + + self._stopped = True + self._channel = TCPClientChannel(self.address, unix=True) + + def _fetch(self) -> None: + """ + Fetch the messages from the outqueue and send them. + + :return: None + """ + while not self._stopped: + try: + msg = self.out_queue.get(block=True, timeout=2.0) + self.send(msg) + except queue.Empty: + pass + + def _receive_loop(self): + """Receive messages.""" + while not self._stopped: + try: + data = self._channel.recv(block=True, timeout=2.0) + if data is not None: + self.in_queue.put_nowait(data) + except queue.Empty: + pass + + @property + def is_established(self) -> bool: + """Return True if the connection has been established, False otherwise.""" + return not self._stopped + + def connect(self): + """Connect to the local OEF Node.""" + if self._stopped: + self._stopped = False + self._channel.connect() + self.in_thread = Thread(target=self._receive_loop) + self.out_thread = Thread(target=self._fetch) + self.in_thread.start() + self.out_thread.start() + + def disconnect(self): + """Disconnect from the local OEF Node.""" + if not self._stopped: + self._stopped = True + self.in_thread.join() + self.out_thread.join() + self.in_thread = None + self.out_thread = None + + def send(self, envelope: Envelope): + """Send a message.""" + if not self.is_established: + raise ConnectionError("Connection not established yet. Please use 'connect()'.") + self._channel.send(envelope) + + def stop(self): + """Tear down the connection.""" + self._channel.disconnect() + + @classmethod + def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': + """Get the Local OEF connection from the connection configuration. + + :param public_key: the public key of the agent. + :param connection_configuration: the connection configuration object. + :return: the connection object + """ + return TCPClientConnection(connection_configuration.config.get("address")) diff --git a/aea/connections/tcp_client/connection.yaml b/aea/connections/tcp_client/connection.yaml new file mode 100644 index 0000000000..040af832a1 --- /dev/null +++ b/aea/connections/tcp_client/connection.yaml @@ -0,0 +1,15 @@ +name: tcp +authors: Fetch.AI Limited +version: 0.1.0 +license: Apache 2.0 +url: "" +class_name: TCPConnection +supported_protocols: + - oef + - default + - fipa + - gym + - tac +config: + addr: 127.0.0.1 + port: 10000 \ No newline at end of file diff --git a/aea/connections/tcp_server/__init__.py b/aea/connections/tcp_server/__init__.py new file mode 100644 index 0000000000..299b9a9d16 --- /dev/null +++ b/aea/connections/tcp_server/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP connection (server side).""" diff --git a/aea/connections/tcp_server/connection.py b/aea/connections/tcp_server/connection.py new file mode 100644 index 0000000000..b0c07b483a --- /dev/null +++ b/aea/connections/tcp_server/connection.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP server.""" +import asyncio +import logging +import queue +import struct +from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, transports +from threading import Thread +from typing import Dict, Optional, Tuple + +from aea.configurations.base import ConnectionConfig +from aea.connections.base import Channel, Connection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + +STUB_DIALOGUE_ID = 0 + + +class TCPServerChannel(Channel): + """Channel implementation for the local node.""" + + def __init__(self, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): + """ + Initialize a TCP channel. + + :param address: the socket bind address. + :param unix: whether it's a unix server or a networked. + :param loop: the asyncio loop. + """ + self.in_queue = queue.Queue() + self.address = address + self.unix = unix + self._loop = asyncio.new_event_loop() if loop is None else loop + + self._thread = None # type: Optional[Thread] + self._server = None # type: Optional[AbstractServer] + self._server_task = None # type: Optional[Task] + + self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] + + async def handle(self, reader: StreamReader, writer: StreamWriter): + public_key = self._recv(reader) + self.connections[public_key] = (reader, writer) + await self._recv_loop(reader, public_key) + + async def _recv(self, reader: StreamReader): + data = await reader.read(len(struct.pack("I", 0))) + nbytes = struct.unpack("I", data)[0] + nbytes_read = 0 + data = b"" + while nbytes_read < nbytes: + data += (await reader.read(nbytes - nbytes_read)) + nbytes_read = len(data) + + return data + + async def _send(self, writer: StreamWriter, data: bytes): + nbytes = struct.pack("I", len(data)) + await writer.write(nbytes) + await writer.write(data) + await writer.drain() + + async def _recv_loop(self, reader, public_key): + data = await self._recv(reader) + envelope = Envelope.decode(data) + self.in_queue.put_nowait(envelope) + await self._recv_loop(reader, public_key) + + def connect(self): + """ + Set up the connection. + + :return: A queue or None. + """ + self._thread = Thread(target=self._loop.run_forever) + self._thread.start() + + if self.unix: + coro = asyncio.start_unix_server(self.handle, self.address, loop=self._loop, start_serving=False) + else: + coro = asyncio.start_server(self.handle, self.address, loop=self._loop, start_serving=False) + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + self._server = future.result() + self._server_task = asyncio.run_coroutine_threadsafe(self._server.serve_forever(), loop=self._loop) + + def disconnect(self) -> None: + """ + Tear down the connection. + + :return: None. + """ + self._server.close() + self._loop.stop() + + def send(self, envelope: Envelope) -> None: + """ + Send an envelope. + + :param envelope: the envelope to send. + :return: None. + """ + to = envelope.to + _, writer = self.connections[to] + asyncio.run_coroutine_threadsafe(self._send(writer, envelope.encode()), loop=self._loop) + + def recv(self, block=True, timeout=None) -> Optional[Envelope]: + """Receive an envelope.""" + try: + self.in_queue.get(block=block, timeout=timeout) + except queue.Empty: + return None + + +class TCPServerConnection(Connection): + """Implementation of a TCP server connection.""" + + def __init__(self, address: str): + """ + Initialize a TCP connection + + :param address: the address + """ + super().__init__() + self.address = address + + self._stopped = True + self._channel = TCPServerChannel(self.address, unix=True) + + def _fetch(self) -> None: + """ + Fetch the messages from the outqueue and send them. + + :return: None + """ + while not self._stopped: + try: + msg = self.out_queue.get(block=True, timeout=2.0) + self.send(msg) + except queue.Empty: + pass + + def _receive_loop(self): + """Receive messages.""" + while not self._stopped: + try: + data = self._channel.recv(block=True, timeout=2.0) + if data is not None: + self.in_queue.put_nowait(data) + except queue.Empty: + pass + + @property + def is_established(self) -> bool: + """Return True if the connection has been established, False otherwise.""" + return not self._stopped + + def connect(self): + """Connect to the local OEF Node.""" + if self._stopped: + self._stopped = False + self._channel.connect() + self.in_thread = Thread(target=self._receive_loop) + self.out_thread = Thread(target=self._fetch) + self.in_thread.start() + self.out_thread.start() + + def disconnect(self): + """Disconnect from the local OEF Node.""" + if not self._stopped: + self._stopped = True + self.in_thread.join() + self.out_thread.join() + self.in_thread = None + self.out_thread = None + + def send(self, envelope: Envelope): + """Send a message.""" + if not self.is_established: + raise ConnectionError("Connection not established yet. Please use 'connect()'.") + self._channel.send(envelope) + + def stop(self): + """Tear down the connection.""" + self._channel.disconnect() + + @classmethod + def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': + """Get the Local OEF connection from the connection configuration. + + :param public_key: the public key of the agent. + :param connection_configuration: the connection configuration object. + :return: the connection object + """ + return TCPServerConnection(connection_configuration.config.get("address")) diff --git a/aea/connections/tcp_server/connection.yaml b/aea/connections/tcp_server/connection.yaml new file mode 100644 index 0000000000..040af832a1 --- /dev/null +++ b/aea/connections/tcp_server/connection.yaml @@ -0,0 +1,15 @@ +name: tcp +authors: Fetch.AI Limited +version: 0.1.0 +license: Apache 2.0 +url: "" +class_name: TCPConnection +supported_protocols: + - oef + - default + - fipa + - gym + - tac +config: + addr: 127.0.0.1 + port: 10000 \ No newline at end of file From a074b39c32ff410aa02eecade4a3cd98e8a3bf1d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 11 Oct 2019 13:29:29 +0200 Subject: [PATCH 02/38] add first prototype of TCP connection. Add TCPChannel and TCPConnection abstraction. The concrete classes are: - TCPServerChannel and TCPServerConnection - TCPClientChannel and TCPClientConnection --- .../{tcp_client => p2p}/__init__.py | 3 +- aea/connections/p2p/base.py | 173 ++++++++++++++ .../__init__.py => p2p/connection.py} | 5 +- .../{tcp_server => p2p}/connection.yaml | 6 +- aea/connections/p2p/tcp_client.py | 124 ++++++++++ aea/connections/p2p/tcp_server.py | 139 ++++++++++++ aea/connections/tcp_client/connection.py | 208 ----------------- aea/connections/tcp_client/connection.yaml | 15 -- aea/connections/tcp_server/connection.py | 214 ------------------ tests/test_connections/test_p2p.py | 74 ++++++ 10 files changed, 518 insertions(+), 443 deletions(-) rename aea/connections/{tcp_client => p2p}/__init__.py (93%) create mode 100644 aea/connections/p2p/base.py rename aea/connections/{tcp_server/__init__.py => p2p/connection.py} (83%) rename aea/connections/{tcp_server => p2p}/connection.yaml (53%) create mode 100644 aea/connections/p2p/tcp_client.py create mode 100644 aea/connections/p2p/tcp_server.py delete mode 100644 aea/connections/tcp_client/connection.py delete mode 100644 aea/connections/tcp_client/connection.yaml delete mode 100644 aea/connections/tcp_server/connection.py create mode 100644 tests/test_connections/test_p2p.py diff --git a/aea/connections/tcp_client/__init__.py b/aea/connections/p2p/__init__.py similarity index 93% rename from aea/connections/tcp_client/__init__.py rename to aea/connections/p2p/__init__.py index d6c337b78a..21d4686b60 100644 --- a/aea/connections/tcp_client/__init__.py +++ b/aea/connections/p2p/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # ------------------------------------------------------------------------------ # # Copyright 2018-2019 Fetch.AI Limited @@ -18,4 +17,4 @@ # # ------------------------------------------------------------------------------ -"""Implementation of the TCP connection (client side).""" +"""Implementation of a P2P connection.""" diff --git a/aea/connections/p2p/base.py b/aea/connections/p2p/base.py new file mode 100644 index 0000000000..f1d123dcad --- /dev/null +++ b/aea/connections/p2p/base.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Base classes for TCP communication.""" +import logging +import queue +import struct +from abc import ABC +from asyncio import StreamReader, StreamWriter, CancelledError +from queue import Queue +from threading import Thread +from typing import Optional + +from aea.connections.base import Channel, Connection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + + +class TCPChannel(Channel, ABC): + """Abstract TCP channel.""" + + def __init__(self, public_key: str): + """Initialize a TCP Channel.""" + self.in_queue = Queue() # type: Queue + self.public_key = public_key + self._stopped = False + + async def _recv(self, reader: StreamReader) -> Optional[bytes]: + """Receive bytes.""" + try: + data = await reader.read(len(struct.pack("I", 0))) + nbytes = struct.unpack("I", data)[0] + nbytes_read = 0 + data = b"" + while nbytes_read < nbytes: + data += (await reader.read(nbytes - nbytes_read)) + nbytes_read = len(data) + + return data + except CancelledError: + return None + except Exception as e: + logger.exception(e) + return None + + async def _send(self, writer: StreamWriter, data: bytes) -> None: + """Send bytes.""" + logger.debug("Send a message") + nbytes = struct.pack("I", len(data)) + logger.debug("#bytes: {}".format(nbytes)) + try: + writer.write(nbytes) + writer.write(data) + await writer.drain() + except CancelledError: + return None + + async def _recv_loop(self, reader) -> None: + """Process incoming messages.""" + try: + if self._stopped: + logger.debug("Stopped receiving loop.") + return + logger.debug("Waiting for next message...") + data = await self._recv(reader) + if data is None: + return + logger.debug("Message received: {}".format(data)) + envelope = Envelope.decode(data) # TODO handle decoding error + logger.debug("Decoded envelope: {}".format(envelope)) + self.in_queue.put_nowait(envelope) + await self._recv_loop(reader) + except Exception as e: + logger.exception(e) + return + + def recv(self, block=True, timeout=None) -> Optional[Envelope]: + """Receive an envelope.""" + try: + return self.in_queue.get(block=block, timeout=timeout) + except queue.Empty: + return None + + +class TCPConnection(Connection, ABC): + """Abstract TCP connection.""" + + _channel: TCPChannel + + def __init__(self, public_key: str, channel: TCPChannel): + """ + Initialize a TCP connection. + + :param public_key: the public key. + :param channel: the TCP channel. + """ + super().__init__() + + self.public_key = public_key + self._channel = channel + self._stopped = True + + def _fetch(self) -> None: + """Fetch the envelopes from the outqueue and send them.""" + while not self._stopped: + try: + msg = self.out_queue.get(block=True, timeout=2.0) + if msg is not None: + self.send(msg) + except queue.Empty: + pass + + def _receive_loop(self): + """Receive envelopes.""" + while not self._stopped: + try: + data = self._channel.recv(block=True, timeout=2.0) + if data is not None: + self.in_queue.put_nowait(data) + except queue.Empty: + pass + + @property + def is_established(self) -> bool: + """Return True if the connection has been established, False otherwise.""" + return not self._stopped + + def connect(self): + """Connect to the local OEF Node.""" + if self._stopped: + self._stopped = False + self._channel.connect() + self.in_thread = Thread(target=self._receive_loop) + self.out_thread = Thread(target=self._fetch) + self.in_thread.start() + self.out_thread.start() + + def disconnect(self): + """Disconnect from the local OEF Node.""" + if not self._stopped: + self._stopped = True + self._channel.disconnect() + self.in_thread.join() + self.out_thread.join() + self.in_thread = None + self.out_thread = None + + def send(self, envelope: Envelope): + """Send a message.""" + if not self.is_established: + raise ConnectionError("Connection not established yet. Please use 'connect()'.") + self._channel.send(envelope) + + def stop(self): + """Tear down the connection.""" + self._channel.disconnect() diff --git a/aea/connections/tcp_server/__init__.py b/aea/connections/p2p/connection.py similarity index 83% rename from aea/connections/tcp_server/__init__.py rename to aea/connections/p2p/connection.py index 299b9a9d16..08c185e93b 100644 --- a/aea/connections/tcp_server/__init__.py +++ b/aea/connections/p2p/connection.py @@ -18,4 +18,7 @@ # # ------------------------------------------------------------------------------ -"""Implementation of the TCP connection (server side).""" +"""Base classes for P2P communication.""" + +from .tcp_client import TCPClientConnection # noqa: F401 +from .tcp_server import TCPServerConnection # noqa: F401 diff --git a/aea/connections/tcp_server/connection.yaml b/aea/connections/p2p/connection.yaml similarity index 53% rename from aea/connections/tcp_server/connection.yaml rename to aea/connections/p2p/connection.yaml index 040af832a1..8ff67f76ba 100644 --- a/aea/connections/tcp_server/connection.yaml +++ b/aea/connections/p2p/connection.yaml @@ -3,7 +3,7 @@ authors: Fetch.AI Limited version: 0.1.0 license: Apache 2.0 url: "" -class_name: TCPConnection +class_name: TCPClientConnection # this can be eitehr TCPClientConnection or TCPServerConnection supported_protocols: - oef - default @@ -11,5 +11,5 @@ supported_protocols: - gym - tac config: - addr: 127.0.0.1 - port: 10000 \ No newline at end of file + address: /tmp/socket + unix: true \ No newline at end of file diff --git a/aea/connections/p2p/tcp_client.py b/aea/connections/p2p/tcp_client.py new file mode 100644 index 0000000000..a92fdf2e6b --- /dev/null +++ b/aea/connections/p2p/tcp_client.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP server.""" +import asyncio +import logging +from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader +from threading import Thread +from typing import Optional, cast, Tuple + +from aea.configurations.base import ConnectionConfig +from aea.connections.base import Connection +from aea.connections.p2p.base import TCPChannel, TCPConnection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + +STUB_DIALOGUE_ID = 0 + + +class TCPClientChannel(TCPChannel): + """Channel implementation for the local node.""" + + def __init__(self, public_key: str, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): + """ + Initialize a TCP channel. + + :param public_key: the public key. + :param address: the socket bind address. + :param unix: whether it's a unix server or a networked. + :param loop: the asyncio loop. + """ + super().__init__(public_key) + self.address = address + self.unix = unix + self._loop = asyncio.new_event_loop() if loop is None else loop + self._thread = None # type: Optional[Thread] + + self._reader, self._writer = (None, None) # type: Optional[StreamReader], Optional[StreamWriter] + self._read_task = None # type: Optional[Task] + + def connect(self): + """ + Set up the connection. + + :return: A queue or None. + """ + self._thread = Thread(target=self._loop.run_forever) + self._thread.start() + + if self.unix: + coro = asyncio.open_unix_connection(path=self.address, loop=self._loop) + else: + ip, port = self.address.split(":") + port = int(port) + coro = asyncio.open_connection(ip, port, loop=self._loop, start_serving=False) + + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + self._reader, self._writer = future.result() + + public_key_bytes = self.public_key.encode("utf-8") + future = asyncio.run_coroutine_threadsafe(self._send(self._writer, public_key_bytes), loop=self._loop) + future.result() + + asyncio.run_coroutine_threadsafe(self._recv_loop(self._reader), self._loop) + + def disconnect(self) -> None: + """ + Tear down the connection. + + :return: None. + """ + if self._stopped: + return + + self._stopped = True + self._writer = cast(StreamWriter, self._writer) + self._thread = cast(Thread, self._thread) + + self._writer.close() + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread.join() + + def send(self, envelope: Envelope) -> None: + """ + Send an envelope. + + :param envelope: the envelope to send. + :return: None. + """ + self._writer = cast(StreamWriter, self._writer) + asyncio.run_coroutine_threadsafe(self._send(self._writer, envelope.encode()), loop=self._loop) + + +class TCPClientConnection(TCPConnection): + """Implementation of a TCP server connection.""" + + @classmethod + def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': + """Get the Local OEF connection from the connection configuration. + + :param public_key: the public key of the agent. + :param connection_configuration: the connection configuration object. + :return: the connection object + """ + address = cast(str, connection_configuration.config.get("address")) + channel = TCPClientChannel(public_key, address, unix=True) + return TCPClientConnection(public_key, channel) diff --git a/aea/connections/p2p/tcp_server.py b/aea/connections/p2p/tcp_server.py new file mode 100644 index 0000000000..69ece6c081 --- /dev/null +++ b/aea/connections/p2p/tcp_server.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP server.""" +import asyncio +import logging +from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer +from threading import Thread +from typing import Dict, Optional, Tuple, cast + +from aea.configurations.base import ConnectionConfig +from aea.connections.base import Connection +from aea.connections.p2p.base import TCPChannel, TCPConnection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + +STUB_DIALOGUE_ID = 0 + + +class TCPServerChannel(TCPChannel): + """Channel implementation for the local node.""" + + def __init__(self, public_key: str, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): + """ + Initialize a TCP channel. + + :param public_key: public key. + :param address: the socket bind address. + :param unix: whether it's a unix server or a networked. + :param loop: the asyncio loop. + """ + super().__init__(public_key) + self.address = address + self.unix = unix + self._loop = asyncio.new_event_loop() if loop is None else loop + + self._thread = None # type: Optional[Thread] + self._server = None # type: Optional[AbstractServer] + self._server_task = None # type: Optional[Task] + + self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] + + async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: + """ + Handle new connections. + + :param reader: the stream reader. + :param writer: the stream writer. + :return: None + """ + logger.debug("Waiting for client public key...") + public_key_bytes = await self._recv(reader) + if public_key_bytes: + public_key_bytes = cast(bytes, public_key_bytes) + public_key = public_key_bytes.decode("utf-8") + logger.debug("Public key of the client: {}".format(public_key)) + self.connections[public_key] = (reader, writer) + await self._recv_loop(reader) + + def connect(self): + """ + Set up the connection. + + :return: A queue or None. + """ + self._thread = Thread(target=self._loop.run_forever) + self._thread.start() + + if self.unix: + coro = asyncio.start_unix_server(self.handle, self.address, loop=self._loop, start_serving=False) + else: + coro = asyncio.start_server(self.handle, self.address, loop=self._loop, start_serving=False) + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + self._server = future.result() + self._server_task = asyncio.run_coroutine_threadsafe(self._server.serve_forever(), loop=self._loop) + + def disconnect(self) -> None: + """ + Tear down the connection. + + :return: None. + """ + if self._stopped: + return + + self._stopped = True + + self._server = cast(AbstractServer, self._server) + self._thread = cast(Thread, self._thread) + + self._server.close() + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread.join() + + def send(self, envelope: Envelope) -> None: + """ + Send an envelope. + + :param envelope: the envelope to send. + :return: None. + """ + to = envelope.to + _, writer = self.connections[to] + future = asyncio.run_coroutine_threadsafe(self._send(writer, envelope.encode()), loop=self._loop) + future.result() + + +class TCPServerConnection(TCPConnection): + """Implementation of a TCP server connection.""" + + @classmethod + def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': + """Get the Local OEF connection from the connection configuration. + + :param public_key: the public key of the agent. + :param connection_configuration: the connection configuration object. + :return: the connection object + """ + address = cast(str, connection_configuration.config.get("address")) + channel = TCPServerChannel(public_key, address, unix=True) + return TCPServerConnection(public_key, channel) diff --git a/aea/connections/tcp_client/connection.py b/aea/connections/tcp_client/connection.py deleted file mode 100644 index cba903693a..0000000000 --- a/aea/connections/tcp_client/connection.py +++ /dev/null @@ -1,208 +0,0 @@ -# -*- coding: utf-8 -*- - -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""Implementation of the TCP server.""" -import asyncio -import logging -import queue -import struct -from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, transports -from ipaddress import IPv4Address -from threading import Thread -from typing import Dict, Optional, Tuple - -from aea.configurations.base import ConnectionConfig -from aea.connections.base import Channel, Connection -from aea.mail.base import Envelope - -logger = logging.getLogger(__name__) - -STUB_DIALOGUE_ID = 0 - - -class TCPClientChannel(Channel): - """Channel implementation for the local node.""" - - def __init__(self, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): - """ - Initialize a TCP channel. - - :param address: the socket bind address. - :param unix: whether it's a unix server or a networked. - :param loop: the asyncio loop. - """ - self.in_queue = queue.Queue() - self.address = address - self.unix = unix - self._loop = asyncio.new_event_loop() if loop is None else loop - self._thread = None # type: Optional[Thread] - - self._reader, self._writer = (None, None) - - async def _recv(self): - data = await self._reader.read(len(struct.pack("I", 0))) - nbytes = struct.unpack("I", data)[0] - nbytes_read = 0 - data = b"" - while nbytes_read < nbytes: - data += (await self._reader.read(nbytes - nbytes_read)) - nbytes_read = len(data) - - return data - - async def _send(self, data: bytes): - nbytes = struct.pack("I", len(data)) - await self._writer.write(nbytes) - await self._writer.write(data) - await self._writer.drain() - - async def _recv_loop(self): - data = await self._recv() - envelope = Envelope.decode(data) - self.in_queue.put_nowait(envelope) - await self._recv_loop() - - def connect(self): - """ - Set up the connection. - - :return: A queue or None. - """ - self._thread = Thread(target=self._loop.run_forever) - self._thread.start() - - if self.unix: - coro = asyncio.open_unix_connection(path=self.address, loop=self._loop) - else: - ip, port = self.address.split(":") - port = int(port) - coro = asyncio.open_connection(ip, port, loop=self._loop, start_serving=False) - future = asyncio.run_coroutine_threadsafe(coro, self._loop) - self._reader, self._writer = future.result() - - - def disconnect(self) -> None: - """ - Tear down the connection. - - :return: None. - """ - self._reader.close() - self._writer.close() - self._loop.stop() - - def send(self, envelope: Envelope) -> None: - """ - Send an envelope. - - :param envelope: the envelope to send. - :return: None. - """ - asyncio.run_coroutine_threadsafe(self._send(envelope.encode()), loop=self._loop) - - def recv(self, block=True, timeout=None) -> Optional[Envelope]: - """Receive an envelope.""" - try: - self.in_queue.get(block=block, timeout=timeout) - except queue.Empty: - return None - - -class TCPClientConnection(Connection): - """Implementation of a TCP server connection.""" - - def __init__(self, address: str): - """ - Initialize a TCP connection - - :param address: the address - """ - super().__init__() - self.address = address - - self._stopped = True - self._channel = TCPClientChannel(self.address, unix=True) - - def _fetch(self) -> None: - """ - Fetch the messages from the outqueue and send them. - - :return: None - """ - while not self._stopped: - try: - msg = self.out_queue.get(block=True, timeout=2.0) - self.send(msg) - except queue.Empty: - pass - - def _receive_loop(self): - """Receive messages.""" - while not self._stopped: - try: - data = self._channel.recv(block=True, timeout=2.0) - if data is not None: - self.in_queue.put_nowait(data) - except queue.Empty: - pass - - @property - def is_established(self) -> bool: - """Return True if the connection has been established, False otherwise.""" - return not self._stopped - - def connect(self): - """Connect to the local OEF Node.""" - if self._stopped: - self._stopped = False - self._channel.connect() - self.in_thread = Thread(target=self._receive_loop) - self.out_thread = Thread(target=self._fetch) - self.in_thread.start() - self.out_thread.start() - - def disconnect(self): - """Disconnect from the local OEF Node.""" - if not self._stopped: - self._stopped = True - self.in_thread.join() - self.out_thread.join() - self.in_thread = None - self.out_thread = None - - def send(self, envelope: Envelope): - """Send a message.""" - if not self.is_established: - raise ConnectionError("Connection not established yet. Please use 'connect()'.") - self._channel.send(envelope) - - def stop(self): - """Tear down the connection.""" - self._channel.disconnect() - - @classmethod - def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': - """Get the Local OEF connection from the connection configuration. - - :param public_key: the public key of the agent. - :param connection_configuration: the connection configuration object. - :return: the connection object - """ - return TCPClientConnection(connection_configuration.config.get("address")) diff --git a/aea/connections/tcp_client/connection.yaml b/aea/connections/tcp_client/connection.yaml deleted file mode 100644 index 040af832a1..0000000000 --- a/aea/connections/tcp_client/connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: tcp -authors: Fetch.AI Limited -version: 0.1.0 -license: Apache 2.0 -url: "" -class_name: TCPConnection -supported_protocols: - - oef - - default - - fipa - - gym - - tac -config: - addr: 127.0.0.1 - port: 10000 \ No newline at end of file diff --git a/aea/connections/tcp_server/connection.py b/aea/connections/tcp_server/connection.py deleted file mode 100644 index b0c07b483a..0000000000 --- a/aea/connections/tcp_server/connection.py +++ /dev/null @@ -1,214 +0,0 @@ -# -*- coding: utf-8 -*- - -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""Implementation of the TCP server.""" -import asyncio -import logging -import queue -import struct -from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, transports -from threading import Thread -from typing import Dict, Optional, Tuple - -from aea.configurations.base import ConnectionConfig -from aea.connections.base import Channel, Connection -from aea.mail.base import Envelope - -logger = logging.getLogger(__name__) - -STUB_DIALOGUE_ID = 0 - - -class TCPServerChannel(Channel): - """Channel implementation for the local node.""" - - def __init__(self, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): - """ - Initialize a TCP channel. - - :param address: the socket bind address. - :param unix: whether it's a unix server or a networked. - :param loop: the asyncio loop. - """ - self.in_queue = queue.Queue() - self.address = address - self.unix = unix - self._loop = asyncio.new_event_loop() if loop is None else loop - - self._thread = None # type: Optional[Thread] - self._server = None # type: Optional[AbstractServer] - self._server_task = None # type: Optional[Task] - - self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] - - async def handle(self, reader: StreamReader, writer: StreamWriter): - public_key = self._recv(reader) - self.connections[public_key] = (reader, writer) - await self._recv_loop(reader, public_key) - - async def _recv(self, reader: StreamReader): - data = await reader.read(len(struct.pack("I", 0))) - nbytes = struct.unpack("I", data)[0] - nbytes_read = 0 - data = b"" - while nbytes_read < nbytes: - data += (await reader.read(nbytes - nbytes_read)) - nbytes_read = len(data) - - return data - - async def _send(self, writer: StreamWriter, data: bytes): - nbytes = struct.pack("I", len(data)) - await writer.write(nbytes) - await writer.write(data) - await writer.drain() - - async def _recv_loop(self, reader, public_key): - data = await self._recv(reader) - envelope = Envelope.decode(data) - self.in_queue.put_nowait(envelope) - await self._recv_loop(reader, public_key) - - def connect(self): - """ - Set up the connection. - - :return: A queue or None. - """ - self._thread = Thread(target=self._loop.run_forever) - self._thread.start() - - if self.unix: - coro = asyncio.start_unix_server(self.handle, self.address, loop=self._loop, start_serving=False) - else: - coro = asyncio.start_server(self.handle, self.address, loop=self._loop, start_serving=False) - future = asyncio.run_coroutine_threadsafe(coro, self._loop) - self._server = future.result() - self._server_task = asyncio.run_coroutine_threadsafe(self._server.serve_forever(), loop=self._loop) - - def disconnect(self) -> None: - """ - Tear down the connection. - - :return: None. - """ - self._server.close() - self._loop.stop() - - def send(self, envelope: Envelope) -> None: - """ - Send an envelope. - - :param envelope: the envelope to send. - :return: None. - """ - to = envelope.to - _, writer = self.connections[to] - asyncio.run_coroutine_threadsafe(self._send(writer, envelope.encode()), loop=self._loop) - - def recv(self, block=True, timeout=None) -> Optional[Envelope]: - """Receive an envelope.""" - try: - self.in_queue.get(block=block, timeout=timeout) - except queue.Empty: - return None - - -class TCPServerConnection(Connection): - """Implementation of a TCP server connection.""" - - def __init__(self, address: str): - """ - Initialize a TCP connection - - :param address: the address - """ - super().__init__() - self.address = address - - self._stopped = True - self._channel = TCPServerChannel(self.address, unix=True) - - def _fetch(self) -> None: - """ - Fetch the messages from the outqueue and send them. - - :return: None - """ - while not self._stopped: - try: - msg = self.out_queue.get(block=True, timeout=2.0) - self.send(msg) - except queue.Empty: - pass - - def _receive_loop(self): - """Receive messages.""" - while not self._stopped: - try: - data = self._channel.recv(block=True, timeout=2.0) - if data is not None: - self.in_queue.put_nowait(data) - except queue.Empty: - pass - - @property - def is_established(self) -> bool: - """Return True if the connection has been established, False otherwise.""" - return not self._stopped - - def connect(self): - """Connect to the local OEF Node.""" - if self._stopped: - self._stopped = False - self._channel.connect() - self.in_thread = Thread(target=self._receive_loop) - self.out_thread = Thread(target=self._fetch) - self.in_thread.start() - self.out_thread.start() - - def disconnect(self): - """Disconnect from the local OEF Node.""" - if not self._stopped: - self._stopped = True - self.in_thread.join() - self.out_thread.join() - self.in_thread = None - self.out_thread = None - - def send(self, envelope: Envelope): - """Send a message.""" - if not self.is_established: - raise ConnectionError("Connection not established yet. Please use 'connect()'.") - self._channel.send(envelope) - - def stop(self): - """Tear down the connection.""" - self._channel.disconnect() - - @classmethod - def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': - """Get the Local OEF connection from the connection configuration. - - :param public_key: the public key of the agent. - :param connection_configuration: the connection configuration object. - :return: the connection object - """ - return TCPServerConnection(connection_configuration.config.get("address")) diff --git a/tests/test_connections/test_p2p.py b/tests/test_connections/test_p2p.py new file mode 100644 index 0000000000..fd78699e1d --- /dev/null +++ b/tests/test_connections/test_p2p.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests of the P2P channel.""" +import logging +import shutil +from pathlib import Path + +from aea.connections.p2p.tcp_client import TCPClientChannel, TCPClientConnection +from aea.connections.p2p.tcp_server import TCPServerConnection, TCPServerChannel +from aea.mail.base import MailBox, Envelope +from aea.protocols.default.message import DefaultMessage +from aea.protocols.default.serialization import DefaultSerializer + + +class TestTCP: + """Test TCP connections.""" + + @classmethod + def setup_class(cls): + """Set up the test class.""" + p = Path("/tmp/aea/test_tcp/") + shutil.rmtree(str(p)) + p.mkdir(parents=True) + + socket_path = str(Path(p, "test_socket")) + cls.server_pbk, cls.client_pbk = "server_pbk", "client_pbk" + server_conn = TCPServerConnection(cls.server_pbk, TCPServerChannel(cls.server_pbk, socket_path, unix=True)) + client_conn = TCPClientConnection(cls.client_pbk, TCPClientChannel(cls.client_pbk, socket_path, unix=True)) + + cls.server_mailbox = MailBox(server_conn) + cls.client_mailbox = MailBox(client_conn) + + cls.server_mailbox.connect() + cls.client_mailbox.connect() + + def test_communication(self): + """Test that we are able to send an envelope from client to server.""" + msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello server") + expected_envelope = Envelope(to=self.server_pbk, sender=self.client_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) + self.client_mailbox.outbox.put(expected_envelope) + actual_envelope = self.server_mailbox.inbox.get(timeout=2.0) + logging.debug(actual_envelope) + assert expected_envelope == actual_envelope + + msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello client") + expected_envelope = Envelope(to=self.client_pbk, sender=self.server_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) + self.server_mailbox.outbox.put(expected_envelope) + actual_envelope = self.client_mailbox.inbox.get(timeout=30.0) + logging.debug(actual_envelope) + assert expected_envelope == actual_envelope + + + @classmethod + def teardown_class(cls): + """Tear down the test class.""" + cls.server_mailbox.disconnect() + cls.client_mailbox.disconnect() From 6a62898aa4a88245e721173db9ef0391f635d7fc Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 11 Oct 2019 13:43:26 +0200 Subject: [PATCH 03/38] restore oef connection.yaml --- aea/connections/oef/connection.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/aea/connections/oef/connection.yaml b/aea/connections/oef/connection.yaml index 0f83c02326..7b040f3707 100644 --- a/aea/connections/oef/connection.yaml +++ b/aea/connections/oef/connection.yaml @@ -6,5 +6,8 @@ url: "" class_name: OEFConnection supported_protocols: ["oef"] config: - bind: /tmp/socket -dependencies: [] \ No newline at end of file + addr: 127.0.0.1 + port: 10000 +dependencies: + - colorlog + - oef \ No newline at end of file From f3bab5b5edb0275b15a8b88a98481d6d9cbb38d2 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 11 Oct 2019 14:30:12 +0200 Subject: [PATCH 04/38] fix tests. --- aea/connections/p2p/tcp_client.py | 2 +- tests/test_connections/test_p2p.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aea/connections/p2p/tcp_client.py b/aea/connections/p2p/tcp_client.py index a92fdf2e6b..e82a2d2a02 100644 --- a/aea/connections/p2p/tcp_client.py +++ b/aea/connections/p2p/tcp_client.py @@ -22,7 +22,7 @@ import logging from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader from threading import Thread -from typing import Optional, cast, Tuple +from typing import Optional, cast from aea.configurations.base import ConnectionConfig from aea.connections.base import Connection diff --git a/tests/test_connections/test_p2p.py b/tests/test_connections/test_p2p.py index fd78699e1d..36cca7b81c 100644 --- a/tests/test_connections/test_p2p.py +++ b/tests/test_connections/test_p2p.py @@ -36,7 +36,7 @@ class TestTCP: def setup_class(cls): """Set up the test class.""" p = Path("/tmp/aea/test_tcp/") - shutil.rmtree(str(p)) + shutil.rmtree(str(p), ignore_errors=True) p.mkdir(parents=True) socket_path = str(Path(p, "test_socket")) From 37c2d7b046872510bcaa7204fda49d52459bf2c0 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 11 Oct 2019 14:36:30 +0200 Subject: [PATCH 05/38] fix code style in test_p2p. --- tests/test_connections/test_p2p.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_connections/test_p2p.py b/tests/test_connections/test_p2p.py index 36cca7b81c..33437e472b 100644 --- a/tests/test_connections/test_p2p.py +++ b/tests/test_connections/test_p2p.py @@ -66,7 +66,6 @@ def test_communication(self): logging.debug(actual_envelope) assert expected_envelope == actual_envelope - @classmethod def teardown_class(cls): """Tear down the test class.""" From 9c20c376d8021cb0aff12e5bdf190f5fd73df62d Mon Sep 17 00:00:00 2001 From: Diarmid Campbell Date: Thu, 31 Oct 2019 11:36:57 +0000 Subject: [PATCH 06/38] post pydata meetup demo --- packages/skills/carpark_client/behaviours.py | 7 +++- packages/skills/carpark_client/dialogues.py | 4 +- packages/skills/carpark_client/handlers.py | 11 ++++-- packages/skills/carpark_client/skill.yaml | 4 +- packages/skills/carpark_client/strategy.py | 14 ++++++- .../skills/carpark_detection/behaviours.py | 1 + .../carpark_detection/detection_database.py | 7 ++-- packages/skills/carpark_detection/handlers.py | 37 +++++++++++++++---- packages/skills/carpark_detection/skill.yaml | 6 +-- packages/skills/carpark_detection/strategy.py | 23 ++++++++++-- 10 files changed, 89 insertions(+), 25 deletions(-) diff --git a/packages/skills/carpark_client/behaviours.py b/packages/skills/carpark_client/behaviours.py index ecab2383eb..e05de4d690 100644 --- a/packages/skills/carpark_client/behaviours.py +++ b/packages/skills/carpark_client/behaviours.py @@ -66,6 +66,7 @@ def act(self) -> None: strategy = cast(Strategy, self.context.strategy) if strategy.is_searching and strategy.is_time_to_search(): self._search_id += 1 + strategy.pause_search() strategy.last_search_time = datetime.datetime.now() query = strategy.get_service_query() search_request = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, @@ -104,7 +105,11 @@ def act(self) -> None: :return: None """ + if not self.context.message_in_queue.empty(): + logger.info("[{}]: self.context.message_in_queue.empty().".format(self.context.agent_name)) if not self._received_tx_message and not self.context.message_in_queue.empty(): + strategy = cast(Strategy, self.context.strategy) + strategy.unpause_search() tx_msg_response = self.context.message_in_queue.get_nowait() if tx_msg_response is not None and \ TransactionMessage.Performative(tx_msg_response.get("performative")) == TransactionMessage.Performative.ACCEPT: @@ -129,7 +134,7 @@ def act(self) -> None: protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(inform_msg)) logger.info("[{}]: informing counterparty={} of transaction digest.".format(self.context.agent_name, counterparty_pbk[-5:])) - self._received_tx_message = True + #self._received_tx_message = True else: logger.info("[{}]: transaction was not successful.".format(self.context.agent_name)) diff --git a/packages/skills/carpark_client/dialogues.py b/packages/skills/carpark_client/dialogues.py index 7288214419..f4d0e5d2a5 100644 --- a/packages/skills/carpark_client/dialogues.py +++ b/packages/skills/carpark_client/dialogues.py @@ -28,6 +28,7 @@ from enum import Enum import logging +import random from typing import Any, Dict, Optional, cast from aea.helpers.dialogue.base import DialogueLabel @@ -178,7 +179,8 @@ def _next_dialogue_id(self) -> int: :return: the next id """ - self._dialogue_id += 1 + self._dialogue_id = random.randint(0, 1000000) + print("_next_dialogue_id: _dialogue_id = {}".format(self._dialogue_id)) return self._dialogue_id def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> bool: diff --git a/packages/skills/carpark_client/handlers.py b/packages/skills/carpark_client/handlers.py index e711337c9f..a54216d4a1 100644 --- a/packages/skills/carpark_client/handlers.py +++ b/packages/skills/carpark_client/handlers.py @@ -147,7 +147,6 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog acceptable = strategy.is_acceptable_proposal(proposal) affordable = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) >= cast(int, proposal.values.get('price')) if acceptable and affordable: - strategy.is_searching = False logger.info("[{}]: accepting the proposal from sender={}".format(self.context.agent_name, sender[-5:])) dialogue.proposal = proposal @@ -163,6 +162,7 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog else: logger.info("[{}]: declining the proposal from sender={}".format(self.context.agent_name, sender[-5:])) + strategy.unpause_search() decline_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target_id, @@ -173,6 +173,7 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(decline_msg)) + def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ Handle the decline. @@ -184,6 +185,8 @@ def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialog :param dialogue: the dialogue object :return: None """ + strategy = cast(Strategy, self.context.strategy) + strategy.unpause_search() logger.info("[{}]: received DECLINE from sender={}".format(self.context.agent_name, sender[-5:])) # target = msg.get("target") # dialogues = cast(Dialogues, self.context.dialogues) @@ -288,11 +291,10 @@ def _handle_search(self, agents: List[str]) -> None: :param agents: the agents returned by the search :return: None """ + strategy = cast(Strategy, self.context.strategy) if len(agents) > 0: logger.info("[{}]: found agents={}, stopping search.".format(self.context.agent_name, list(map(lambda x: x[-5:], agents)))) - strategy = cast(Strategy, self.context.strategy) - # stopping search - strategy.is_searching = False + # pick first agent found opponent_pbk = agents[0] dialogues = cast(Dialogues, self.context.dialogues) @@ -310,4 +312,5 @@ def _handle_search(self, agents: List[str]) -> None: protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(cfp_msg)) else: + strategy.unpause_search() logger.info("[{}]: found no agents, continue searching.".format(self.context.agent_name)) diff --git a/packages/skills/carpark_client/skill.yaml b/packages/skills/carpark_client/skill.yaml index 2684ab370f..b8615114dc 100644 --- a/packages/skills/carpark_client/skill.yaml +++ b/packages/skills/carpark_client/skill.yaml @@ -23,8 +23,8 @@ shared_classes: class_name: Strategy args: country: UK - search_interval: 5 - max_price: 4000 + search_interval: 10 + max_price: 400000000 max_detection_age: 3600000 - shared_class: class_name: Dialogues diff --git a/packages/skills/carpark_client/strategy.py b/packages/skills/carpark_client/strategy.py index a0338f92a7..0f97f10a60 100644 --- a/packages/skills/carpark_client/strategy.py +++ b/packages/skills/carpark_client/strategy.py @@ -48,7 +48,8 @@ def __init__(self, **kwargs) -> None: self._max_detection_age = kwargs.pop('max_detection_age') if 'max_detection_age' in kwargs.keys() else DEFAULT_MAX_DETECTION_AGE super().__init__(**kwargs) self.is_searching = True - self.last_search_time = datetime.datetime.now() + self.last_search_time = datetime.datetime.now() - datetime.timedelta(seconds=self._search_interval) + print ("self._max_price = {}".format(self._max_price )) def get_service_query(self) -> Query: """ @@ -59,6 +60,15 @@ def get_service_query(self) -> Query: query = Query([Constraint('longitude', ConstraintType("!=", 0.0))], model=None) return query + def pause_search(self): + """Stop searching temporarily""" + self.is_searching = False + + def unpause_search(self): + """Restart searching after pausing""" + self.last_search_time = datetime.datetime.now() + self.is_searching = True + def is_time_to_search(self) -> bool: """ Check whether it is time to search. @@ -67,6 +77,7 @@ def is_time_to_search(self) -> bool: """ now = datetime.datetime.now() diff = now - self.last_search_time + # print("is_time_to_search: diff = {}".format(diff)) result = diff.total_seconds() > self._search_interval return result @@ -78,5 +89,6 @@ def is_acceptable_proposal(self, proposal: Description) -> bool: """ result = proposal.values["price"] < self._max_price and \ proposal.values["last_detection_time"] > int(time.time()) - self._max_detection_age + print("is_acceptable_proposal price = {} - self._max_price = {}".format(proposal.values["price"], self._max_price)) return result diff --git a/packages/skills/carpark_detection/behaviours.py b/packages/skills/carpark_detection/behaviours.py index 28d809e933..7c01becf33 100644 --- a/packages/skills/carpark_detection/behaviours.py +++ b/packages/skills/carpark_detection/behaviours.py @@ -53,6 +53,7 @@ def setup(self) -> None: :return: None """ balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) + logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, balance)) if not self._registered: strategy = cast(Strategy, self.context.strategy) diff --git a/packages/skills/carpark_detection/detection_database.py b/packages/skills/carpark_detection/detection_database.py index dcb612b47e..6df5ac9ec5 100644 --- a/packages/skills/carpark_detection/detection_database.py +++ b/packages/skills/carpark_detection/detection_database.py @@ -21,14 +21,14 @@ import sqlite3 import os import shutil -import skimage # type: ignore +import skimage import time class DetectionDatabase: """Communicate between the database and the python objects.""" - def __init__(self, temp_dir): + def __init__(self, temp_dir, create_if_not_present=True): """Initialise the Detection Database Communication class.""" self.this_dir = os.path.dirname(__file__) self.temp_dir = temp_dir @@ -43,7 +43,8 @@ def __init__(self, temp_dir): self.image_file_ext = ".png" self.database_path = self.temp_dir + "/" + "detection_results.db" - self.initialise_backend() + if create_if_not_present: + self.initialise_backend() def reset_database(self): """Reset the database and remove all data.""" diff --git a/packages/skills/carpark_detection/handlers.py b/packages/skills/carpark_detection/handlers.py index f1cbb93aa8..0623f1768d 100644 --- a/packages/skills/carpark_detection/handlers.py +++ b/packages/skills/carpark_detection/handlers.py @@ -134,7 +134,8 @@ def _handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_i query = cast(Query, msg.get("query")) strategy = cast(Strategy, self.context.strategy) - if strategy.is_matching_supply(query): + + if strategy.is_matching_supply(query) and strategy.has_data(): proposal, carpark_data = strategy.generate_proposal_and_data(query) dialogue.carpark_data = carpark_data dialogue.proposal = proposal @@ -151,6 +152,9 @@ def _handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_i sender=self.context.agent_public_key, protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(proposal_msg)) + + strategy.db.set_dialogue_status(dialogue_id, sender[-5:], "received_cfp", "send_proposal") + else: logger.info("[{}]: declined the CFP from sender={}".format(self.context.agent_name, sender[-5:])) @@ -164,6 +168,8 @@ def _handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_i protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(decline_msg)) + strategy.db.set_dialogue_status(dialogue_id, sender[-5:], "received_cfp", "send_no_proposal") + def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ Handle the DECLINE. @@ -179,6 +185,8 @@ def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialog """ logger.info("[{}]: received DECLINE from sender={}".format(self.context.agent_name, sender[-5:])) + strategy = cast(Strategy, self.context.strategy) + strategy.db.set_dialogue_status(dialogue_id, sender[-5:], "received_decline", "[NONE]") # dialogues = cast(Dialogues, self.context.dialogues) # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.DECLINED_PROPOSE) @@ -211,6 +219,8 @@ def _handle_accept(self, msg: FIPAMessage, sender: str, message_id: int, dialogu sender=self.context.agent_public_key, protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(match_accept_msg)) + strategy = cast(Strategy, self.context.strategy) + strategy.db.set_dialogue_status(dialogue_id, sender[-5:], "received_accept", "send_match_accept") def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ @@ -240,12 +250,18 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu total_price = cast(int, proposal.values.get("price")) is_settled = self.context.ledger_apis.is_tx_settled('fetchai', tx_digest, total_price) if is_settled: - token_balance = self.context.ledger_apis.token_balance('fetchai', - cast(str, self.context.agent_addresses.get('fetchai'))) - logger.info("[{}]: transaction={} settled, new balance={}. Sending data to sender={}".format(self.context.agent_name, - tx_digest, - token_balance, - sender[-5:])) + token_balance = self.context.ledger_apis.token_balance( + 'fetchai', + cast(str, self.context.agent_addresses.get('fetchai'))) + + strategy = cast(Strategy, self.context.strategy) + strategy.record_balance(token_balance) + + logger.info("[{}]: transaction={} settled, new balance={}. Sending data to sender={}".format( + self.context.agent_name, + tx_digest, + token_balance, + sender[-5:])) inform_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target, @@ -259,6 +275,13 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu message=FIPASerializer().encode(inform_msg)) # dialogues = cast(Dialogues, self.context.dialogues) # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.SUCCESSFUL) + strategy.db.add_in_progress_transaction( + tx_digest, + sender[-5:], + self.context.agent_name, + total_price) + strategy.db.set_transaction_complete(tx_digest) + strategy.db.set_dialogue_status(dialogue_id, sender[-5:], "transaction_complete", "send_request_data") else: logger.info("[{}]: transaction={} not settled, aborting".format(self.context.agent_name, tx_digest)) diff --git a/packages/skills/carpark_detection/skill.yaml b/packages/skills/carpark_detection/skill.yaml index ec9d30cd5d..548d1e1658 100644 --- a/packages/skills/carpark_detection/skill.yaml +++ b/packages/skills/carpark_detection/skill.yaml @@ -16,9 +16,9 @@ shared_classes: - shared_class: class_name: Strategy args: - data_price_fet: 2000 - # db_is_rel_to_cwd: true - # db_rel_dir: temp_files + data_price_fet: 200000000 + db_is_rel_to_cwd: true + db_rel_dir: ../temp_files - shared_class: class_name: Dialogues args: {} diff --git a/packages/skills/carpark_detection/strategy.py b/packages/skills/carpark_detection/strategy.py index 64f8946da5..ecfe4f37ec 100644 --- a/packages/skills/carpark_detection/strategy.py +++ b/packages/skills/carpark_detection/strategy.py @@ -19,7 +19,8 @@ """This module contains the strategy class.""" import os -from typing import Any, Dict, List, Tuple, TYPE_CHECKING +from typing import Any, Dict, List, Tuple, TYPE_CHECKING, cast +import time from aea.protocols.oef.models import Description, Query from aea.skills.base import SharedClass @@ -60,10 +61,19 @@ def __init__(self, **kwargs) -> None: self.data_price_fet = kwargs.pop('data_price_fet') if 'data_price_fet' in kwargs.keys() else DEFAULT_PRICE super().__init__(**kwargs) - self.db = DetectionDatabase(db_dir) - self.data_price_fet = 2000 + balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) + + if not os.path.isdir(db_dir): + print("WARNING - DATABASE dir does not exist") + + self.db = DetectionDatabase(db_dir, False) self.lat = 43 self.lon = 42 + self.record_balance(balance) + + + def record_balance(self, balance): + self.db.set_fet(balance, time.time()) def get_service_description(self) -> Description: """ @@ -91,6 +101,11 @@ def is_matching_supply(self, query: Query) -> bool: # TODO, this is a stub return True + def has_data(self) -> bool: + """Return whether we have any useful data to sell""" + data = self.db.get_latest_detection_data(1) + return len(data) > 0 + def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[str, List[Dict[str, Any]]]]: """ Generate a proposal matching the query. @@ -100,6 +115,7 @@ def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[st """ # TODO, this is a stub data = self.db.get_latest_detection_data(1) + assert (len(data) > 0) del data[0]['raw_image_path'] del data[0]['processed_image_path'] @@ -119,3 +135,4 @@ def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[st data[0]["message_type"] = "car_park_data" return proposal, data[0] + From 7322970cca1d7b0b14907bc7bca9f554b07325a5 Mon Sep 17 00:00:00 2001 From: Diarmid Campbell Date: Thu, 31 Oct 2019 13:47:46 +0000 Subject: [PATCH 07/38] fixed tests --- packages/skills/carpark_client/handlers.py | 1 - packages/skills/carpark_client/strategy.py | 5 ++--- packages/skills/carpark_detection/detection_database.py | 2 +- packages/skills/carpark_detection/handlers.py | 1 - packages/skills/carpark_detection/strategy.py | 5 ++--- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/skills/carpark_client/handlers.py b/packages/skills/carpark_client/handlers.py index ecf5d4cd42..a6c46718d1 100644 --- a/packages/skills/carpark_client/handlers.py +++ b/packages/skills/carpark_client/handlers.py @@ -174,7 +174,6 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(decline_msg)) - def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ Handle the decline. diff --git a/packages/skills/carpark_client/strategy.py b/packages/skills/carpark_client/strategy.py index 4c52dbbfbd..d160c74eba 100644 --- a/packages/skills/carpark_client/strategy.py +++ b/packages/skills/carpark_client/strategy.py @@ -60,11 +60,11 @@ def get_service_query(self) -> Query: return query def pause_search(self): - """Stop searching temporarily""" + """Stop searching temporarily.""" self.is_searching = False def unpause_search(self): - """Restart searching after pausing""" + """Restart searching after pausing.""" self.last_search_time = datetime.datetime.now() self.is_searching = True @@ -76,7 +76,6 @@ def is_time_to_search(self) -> bool: """ now = datetime.datetime.now() diff = now - self.last_search_time - # print("is_time_to_search: diff = {}".format(diff)) result = diff.total_seconds() > self._search_interval return result diff --git a/packages/skills/carpark_detection/detection_database.py b/packages/skills/carpark_detection/detection_database.py index 6df5ac9ec5..102d7c7a7b 100644 --- a/packages/skills/carpark_detection/detection_database.py +++ b/packages/skills/carpark_detection/detection_database.py @@ -21,7 +21,7 @@ import sqlite3 import os import shutil -import skimage +import skimage # type: ignore import time diff --git a/packages/skills/carpark_detection/handlers.py b/packages/skills/carpark_detection/handlers.py index 0623f1768d..da0c1e0e2e 100644 --- a/packages/skills/carpark_detection/handlers.py +++ b/packages/skills/carpark_detection/handlers.py @@ -134,7 +134,6 @@ def _handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_i query = cast(Query, msg.get("query")) strategy = cast(Strategy, self.context.strategy) - if strategy.is_matching_supply(query) and strategy.has_data(): proposal, carpark_data = strategy.generate_proposal_and_data(query) dialogue.carpark_data = carpark_data diff --git a/packages/skills/carpark_detection/strategy.py b/packages/skills/carpark_detection/strategy.py index ecfe4f37ec..3ec1b1cda7 100644 --- a/packages/skills/carpark_detection/strategy.py +++ b/packages/skills/carpark_detection/strategy.py @@ -71,8 +71,8 @@ def __init__(self, **kwargs) -> None: self.lon = 42 self.record_balance(balance) - def record_balance(self, balance): + """Record current balance to database.""" self.db.set_fet(balance, time.time()) def get_service_description(self) -> Description: @@ -102,7 +102,7 @@ def is_matching_supply(self, query: Query) -> bool: return True def has_data(self) -> bool: - """Return whether we have any useful data to sell""" + """Return whether we have any useful data to sell.""" data = self.db.get_latest_detection_data(1) return len(data) > 0 @@ -135,4 +135,3 @@ def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[st data[0]["message_type"] = "car_park_data" return proposal, data[0] - From 71852200ebae64c70f84e473b92958a6b2d827d5 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Thu, 31 Oct 2019 13:48:12 +0000 Subject: [PATCH 08/38] Added tests for crypto - Updated decision_maker_base --- aea/crypto/ledger_apis.py | 20 +----- tests/data/eth_private_key.txt | 2 +- tests/test_crypto/test_helpers.py | 50 +++++++++++++++ tests/test_crypto/test_ledger_apis.py | 89 ++++++++++++++++++++++++++ tests/test_decision_maker/test_base.py | 4 ++ 5 files changed, 147 insertions(+), 18 deletions(-) create mode 100644 tests/test_crypto/test_helpers.py create mode 100644 tests/test_crypto/test_ledger_apis.py diff --git a/aea/crypto/ledger_apis.py b/aea/crypto/ledger_apis.py index a6d92ccf12..6ff1e1c200 100644 --- a/aea/crypto/ledger_apis.py +++ b/aea/crypto/ledger_apis.py @@ -101,7 +101,7 @@ def token_balance(self, identifier: str, address: str) -> int: except Exception: logger.warning("An error occurred while attempting to get the current balance.") balance = 0 - else: + else: # pragma: no cover balance = 0 return balance @@ -156,17 +156,16 @@ def transfer(self, identifier: str, crypto_object: Crypto, destination_address: time.sleep(3.0) return tx_digest - else: + else: # pragma: no cover tx_digest = None return tx_digest - def is_tx_settled(self, identifier: str, tx_digest: str, amount: int) -> bool: + def is_tx_settled(self, identifier: str, tx_digest: str) -> bool: """ Check whether the transaction is settled and correct. :param identifier: the identifier of the ledger :param tx_digest: the transaction digest - :param amount: the amount :return: True if correctly settled, False otherwise """ assert identifier in self.apis.keys(), "Unsupported ledger identifier." @@ -194,16 +193,3 @@ def is_tx_settled(self, identifier: str, tx_digest: str, amount: int) -> bool: logger.warning("An error occured while attempting to check the transaction!") return is_successful - - @staticmethod - def get_address_from_public_key(self, identifier: str, public_key: str) -> Address: - """ - Get the address from the public key. - - :param identifier: the identifier - :param public_key: the public key - :return: the address - """ - assert identifier in self.apis.keys(), "Unsupported ledger identifier." - identity = Identity.from_hex(public_key) - return Address(identity) diff --git a/tests/data/eth_private_key.txt b/tests/data/eth_private_key.txt index 97d4098c2c..09064f89fa 100644 --- a/tests/data/eth_private_key.txt +++ b/tests/data/eth_private_key.txt @@ -1 +1 @@ -0x0c70c25dc9fcb75ae5122eca8a750403ff5420236a6475ac25d9504d6318b6eb \ No newline at end of file +0x6F611408F7EF304947621C51A4B7D84A13A2B9786E9F984DA790A096E8260C64 \ No newline at end of file diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py new file mode 100644 index 0000000000..ae5bbda2eb --- /dev/null +++ b/tests/test_crypto/test_helpers.py @@ -0,0 +1,50 @@ + +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the crypto/helpers module.""" +import os +import pytest + +from aea.crypto.helpers import _try_validate_private_key_pem_path, _try_validate_fet_private_key_path, \ + _try_validate_ethereum_private_key_path +from tests.conftest import CUR_PATH + + +class TestHelperFile: + """Test helper module in aea/crypto.""" + def tests_private_keys(self): + """Test the private keys.""" + private_key_path = os.path.join(CUR_PATH, "data", "priv.pem") + _try_validate_private_key_pem_path(private_key_path) + with pytest.raises(SystemExit): + private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") + _try_validate_private_key_pem_path(private_key_path) + + private_key_path = os.path.join(CUR_PATH, "data", "fet_private_key.txt") + _try_validate_fet_private_key_path(private_key_path) + with pytest.raises(SystemExit): + private_key_path = os.path.join(CUR_PATH, "data", "priv_wrong.pem") + _try_validate_fet_private_key_path(private_key_path) + + private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") + _try_validate_ethereum_private_key_path(private_key_path) + with pytest.raises(SystemExit): + private_key_path = os.path.join(CUR_PATH, "data", "priv_wrong.pem") + _try_validate_ethereum_private_key_path(private_key_path) diff --git a/tests/test_crypto/test_ledger_apis.py b/tests/test_crypto/test_ledger_apis.py new file mode 100644 index 0000000000..7a84d46b19 --- /dev/null +++ b/tests/test_crypto/test_ledger_apis.py @@ -0,0 +1,89 @@ + +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the crypto/helpers module.""" +import logging +import subprocess + +import os + +import pytest + +from aea.crypto.ethereum import ETHEREUM, EthereumCrypto +from aea.crypto.fetchai import FETCHAI, FetchAICrypto +from aea.crypto.ledger_apis import LedgerApis, DEFAULT_FETCHAI_CONFIG +from tests.conftest import CUR_PATH + +logger = logging.getLogger(__name__) + +DEFAULT_ETHEREUM_CONFIG = ("https://ropsten.infura.io/v3/f00f7b3ba0e848ddbdc8941c527447fe", 3) +fet_address = "B3t9pv4rYccWqCjeuoXsDoeXLiKxVAQh6Q3CLAiNZZQ2mtqF1" +eth_address = "0x21795D753752ccC1AC728002D23Ba33cbF13b8b0" + +class TestLedgerApis: + """Test the ledger_apis module.""" + + def test_initialisation(self): + """Test the initialisation of the ledger APIs.""" + ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG}) + assert ledger_apis.configs.get(ETHEREUM) == DEFAULT_ETHEREUM_CONFIG + + unknown_config = ("UknownPath", 8080) + with pytest.raises(ValueError): + ledger_apis = LedgerApis({"UNKNOWN": unknown_config}) + + def test_token_balance(self): + """Test the token_balance for the different tokens.""" + ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, + FETCHAI: DEFAULT_FETCHAI_CONFIG}) + + balance = ledger_apis.token_balance(FETCHAI, eth_address) + assert balance == 0 + balance = ledger_apis.token_balance(ETHEREUM, eth_address) + assert balance != 0, "The specific address has some eth" + balance = ledger_apis.token_balance(ETHEREUM, fet_address) + assert balance == 0, "Should trigger the Exception and the balance will be 0" + with pytest.raises(AssertionError): + balance = ledger_apis.token_balance("UNKNOWN", fet_address) + assert balance == 0, "Unknown identifier so it will return 0" + + def test_transfer(self): + """Test the transfer function for the supported tokens.""" + private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") + eth_obj = EthereumCrypto(private_key_path=private_key_path) + private_key_path = os.path.join(CUR_PATH, 'data', "fet_private_key.txt") + fet_obj = FetchAICrypto(private_key_path=private_key_path) + ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, + FETCHAI: DEFAULT_FETCHAI_CONFIG}) + tx_digest =ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) + assert tx_digest is not None + assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest) + tx_digest = "unknown_hash" + assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest) + + tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=10, tx_fee=200000) + assert tx_digest is not None + assert ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest) + tx_digest = "unknown_hash" + assert not ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest) + with pytest.raises(ValueError): + tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=50, tx_fee=2) + + diff --git a/tests/test_decision_maker/test_base.py b/tests/test_decision_maker/test_base.py index 5e21690060..b6d8d08ad2 100644 --- a/tests/test_decision_maker/test_base.py +++ b/tests/test_decision_maker/test_base.py @@ -296,6 +296,10 @@ def test_decision_maker_execute(self): self.decision_maker.handle(tx_message) assert not self.decision_maker.message_out_queue.empty() + with mock.patch.object(self.decision_maker, "_is_acceptable_tx", return_value=False): + self.decision_maker.handle(tx_message) + assert not self.decision_maker.message_out_queue.empty() + def test_decision_maker_execute_w_wrong_input(self): """Test the execute method with wrong input.""" default_message = DefaultMessage(type=DefaultMessage.Type.BYTES, From d09b053fc234d984f258267fd6a3c80c1a72e67f Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Thu, 31 Oct 2019 14:01:53 +0000 Subject: [PATCH 09/38] Linting --- aea/crypto/ledger_apis.py | 1 - packages/skills/carpark_detection/handlers.py | 8 ++++---- packages/skills/weather_station_ledger/handlers.py | 4 ++-- tests/test_crypto/test_helpers.py | 1 + tests/test_crypto/test_ledger_apis.py | 8 ++------ 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/aea/crypto/ledger_apis.py b/aea/crypto/ledger_apis.py index 6ff1e1c200..6d7efa822e 100644 --- a/aea/crypto/ledger_apis.py +++ b/aea/crypto/ledger_apis.py @@ -27,7 +27,6 @@ import web3 import web3.exceptions from fetchai.ledger.api import LedgerApi as FetchLedgerApi -from fetchai.ledger.crypto import Identity, Address from web3 import Web3, HTTPProvider from aea.crypto.base import Crypto diff --git a/packages/skills/carpark_detection/handlers.py b/packages/skills/carpark_detection/handlers.py index f1cbb93aa8..64a8db8773 100644 --- a/packages/skills/carpark_detection/handlers.py +++ b/packages/skills/carpark_detection/handlers.py @@ -28,7 +28,7 @@ from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer -from aea.protocols.oef.models import Description, Query +from aea.protocols.oef.models import Query # Description from aea.skills.base import Handler if TYPE_CHECKING: @@ -236,9 +236,9 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu tx_digest = json_data['transaction_digest'] logger.info("[{}]: checking whether transaction={} has been received ...".format(self.context.agent_name, tx_digest)) - proposal = cast(Description, dialogue.proposal) - total_price = cast(int, proposal.values.get("price")) - is_settled = self.context.ledger_apis.is_tx_settled('fetchai', tx_digest, total_price) + # proposal = cast(Description, dialogue.proposal) + # total_price = cast(int, proposal.values.get("price")) + is_settled = self.context.ledger_apis.is_tx_settled('fetchai', tx_digest) if is_settled: token_balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) diff --git a/packages/skills/weather_station_ledger/handlers.py b/packages/skills/weather_station_ledger/handlers.py index ee248c4d14..603402c6e7 100644 --- a/packages/skills/weather_station_ledger/handlers.py +++ b/packages/skills/weather_station_ledger/handlers.py @@ -239,9 +239,9 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu logger.info("[{}]: checking whether transaction={} has been received ...".format(self.context.agent_name, tx_digest)) proposal = cast(Description, dialogue.proposal) - total_price = cast(int, proposal.values.get("price")) + # total_price = cast(int, proposal.values.get("price")) ledger_id = cast(str, proposal.values.get("ledger_id")) - is_settled = self.context.ledger_apis.is_tx_settled(ledger_id, tx_digest, total_price) + is_settled = self.context.ledger_apis.is_tx_settled(ledger_id, tx_digest) if is_settled: token_balance = self.context.ledger_apis.token_balance(ledger_id, cast(str, self.context.agent_addresses.get(ledger_id))) diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index ae5bbda2eb..51a2ca0abb 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -29,6 +29,7 @@ class TestHelperFile: """Test helper module in aea/crypto.""" + def tests_private_keys(self): """Test the private keys.""" private_key_path = os.path.join(CUR_PATH, "data", "priv.pem") diff --git a/tests/test_crypto/test_ledger_apis.py b/tests/test_crypto/test_ledger_apis.py index 7a84d46b19..b238604044 100644 --- a/tests/test_crypto/test_ledger_apis.py +++ b/tests/test_crypto/test_ledger_apis.py @@ -20,10 +20,7 @@ """This module contains the tests for the crypto/helpers module.""" import logging -import subprocess - import os - import pytest from aea.crypto.ethereum import ETHEREUM, EthereumCrypto @@ -37,6 +34,7 @@ fet_address = "B3t9pv4rYccWqCjeuoXsDoeXLiKxVAQh6Q3CLAiNZZQ2mtqF1" eth_address = "0x21795D753752ccC1AC728002D23Ba33cbF13b8b0" + class TestLedgerApis: """Test the ledger_apis module.""" @@ -72,7 +70,7 @@ def test_transfer(self): fet_obj = FetchAICrypto(private_key_path=private_key_path) ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, FETCHAI: DEFAULT_FETCHAI_CONFIG}) - tx_digest =ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) + tx_digest = ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) assert tx_digest is not None assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest) tx_digest = "unknown_hash" @@ -85,5 +83,3 @@ def test_transfer(self): assert not ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest) with pytest.raises(ValueError): tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=50, tx_fee=2) - - From 20e81d356f1bf36248f5dad34950cab8260ea2bb Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Oct 2019 17:20:28 +0000 Subject: [PATCH 10/38] Streamline weather skills with ledger and dialogue management --- aea/crypto/ledger_apis.py | 10 ++ aea/protocols/fipa/message.py | 17 ++- examples/weather_skills/README.md | 75 ++++++++--- .../weather_client_ledger/behaviours.py | 53 ++++---- .../skills/weather_client_ledger/dialogues.py | 119 ++++-------------- .../skills/weather_client_ledger/handlers.py | 31 ++--- .../skills/weather_client_ledger/skill.yaml | 5 +- .../skills/weather_client_ledger/strategy.py | 44 ++++++- .../weather_station_ledger/behaviours.py | 38 ++++-- .../weather_station_ledger/dialogues.py | 115 ++++++----------- .../skills/weather_station_ledger/handlers.py | 4 +- .../skills/weather_station_ledger/skill.yaml | 5 +- .../skills/weather_station_ledger/strategy.py | 35 ++++-- 13 files changed, 282 insertions(+), 269 deletions(-) diff --git a/aea/crypto/ledger_apis.py b/aea/crypto/ledger_apis.py index a6d92ccf12..0188d4ea2f 100644 --- a/aea/crypto/ledger_apis.py +++ b/aea/crypto/ledger_apis.py @@ -79,6 +79,16 @@ def apis(self) -> Dict[str, Any]: """Get the apis.""" return self._apis + @property + def has_fetchai(self): + """Check if it has the fetchai API.""" + return FETCHAI in self.apis.keys() + + @property + def has_ethereum(self): + """Check if it has the ethereum API.""" + return ETHEREUM in self.apis.keys() + def token_balance(self, identifier: str, address: str) -> int: """ Get the token balance. diff --git a/aea/protocols/fipa/message.py b/aea/protocols/fipa/message.py index e3d617e8d1..de0bfc267b 100644 --- a/aea/protocols/fipa/message.py +++ b/aea/protocols/fipa/message.py @@ -20,7 +20,7 @@ """This module contains the FIPA message definition.""" from enum import Enum -from typing import Optional, Union +from typing import Dict, List, Optional, Union from aea.protocols.base import Message from aea.protocols.oef.models import Description, Query @@ -31,6 +31,9 @@ class FIPAMessage(Message): protocol_id = "fipa" + STARTING_MESSAGE_ID = 1 + STARTING_TARGET = 0 + class Performative(Enum): """FIPA performatives.""" @@ -100,3 +103,15 @@ def check_consistency(self) -> bool: return False return True + + +VALID_PREVIOUS_PERFORMATIVES = { + FIPAMessage.Performative.CFP: [None], + FIPAMessage.Performative.PROPOSE: [FIPAMessage.Performative.CFP], + FIPAMessage.Performative.ACCEPT: [FIPAMessage.Performative.PROPOSE], + FIPAMessage.Performative.ACCEPT_W_ADDRESS: [FIPAMessage.Performative.PROPOSE], + FIPAMessage.Performative.MATCH_ACCEPT: [FIPAMessage.Performative.ACCEPT, FIPAMessage.Performative.ACCEPT_W_ADDRESS], + FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS: [FIPAMessage.Performative.ACCEPT, FIPAMessage.Performative.ACCEPT_W_ADDRESS], + FIPAMessage.Performative.INFORM: [FIPAMessage.Performative.MATCH_ACCEPT, FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS, FIPAMessage.Performative.INFORM], + FIPAMessage.Performative.DECLINE: [FIPAMessage.Performative.CFP, FIPAMessage.Performative.PROPOSE, FIPAMessage.Performative.ACCEPT, FIPAMessage.Performative.ACCEPT_W_ADDRESS] +} # type: Dict[FIPAMessage.Performative, List[Union[None, FIPAMessage.Performative]]] diff --git a/examples/weather_skills/README.md b/examples/weather_skills/README.md index ed99d1bd4f..51e624f1c4 100644 --- a/examples/weather_skills/README.md +++ b/examples/weather_skills/README.md @@ -1,27 +1,28 @@ # Weather station and client example A guide to create two AEAs, one a weather station selling weather data, another a -purchaser (client) of weather data. The AEAs use the Fetch.ai ledger to settle their -trade. This setup assumes the weather client trusts the weather station to send the data -upon successful payment. +purchaser (client) of weather data. This setup assumes the weather client trusts the weather station +to send the data upon successful payment. -## Quick start +## Without ledger -- Launch the OEF Node: +The AEAs negotiate and then transfer the data. No payment takes place. + +- Launch the OEF Node (for search and discovery): python scripts/oef/launch.py -c ./scripts/oef/launch_config.json - Create a weather station - the agent that will provide weather measurements: - aea create weather_station + aea create weather_station cd weather_station aea add skill weather_station aea run - In another terminal, create the weather client - the agent that will query the weather station - aea create weather_client - cd weather_client + aea create weather_client + cd weather_client aea add skill weather_client aea run @@ -32,24 +33,23 @@ upon successful payment. aea delete weather_client -## Using the ledger +## With ledger (Fetch.ai) -To run the same example but with a true ledger transaction, -follow these steps: +The AEAs use the Fetch.ai ledger to settle their trade. -- Launch the OEF Node: +- Launch the OEF Node (for search and discovery): python scripts/oef/launch.py -c ./scripts/oef/launch_config.json - Create a weather station (ledger version) - the agent that will provide weather measurements: - aea create weather_station + aea create weather_station cd weather_station aea add skill weather_station_ledger - In another terminal, create the weather client (ledger version) - the agent that will query the weather station - aea create weather_client + aea create weather_client cd weather_client aea add skill weather_client_ledger @@ -65,10 +65,6 @@ ledger_apis: addr: alpha.fetch-ai.com ledger: fetchai port: 80 -- ledger_api: - addr: https://ropsten.infura.io/v3/f00f7b3ba0e848ddbdc8941c527447fe - ledger: ethereum - port: 3 ``` - Generate some wealth to your weather client FET address (it takes a while): @@ -78,8 +74,47 @@ python scripts/fetchai_wealth_generation.py --private-key weather_client/fet_pri cd weather_client ``` +- Run both agents, from their respective terminals: + + aea run + +- Afterwards, clean up: + + cd .. + aea delete weather_station + aea delete weather_client + +## With ledger (Ethereum) + +The AEAs use the Ethereum ledger to settle their trade. + +- Follow the first three steps from the previous section. + +- Generate the private key for the weather client: + + aea generate-key ethereum + +- Both in `weather_station/aea-config.yaml` and +`weather_client/aea-config.yaml`, replace `ledger_apis: []` with: +``` +- ledger_api: + addr: https://ropsten.infura.io/v3/f00f7b3ba0e848ddbdc8941c527447fe + ledger: ethereum + port: 3 +``` + +- In weather station client skill config under strategy change to +``` +currency_pbk: 'ETH' +ledger_id: 'ethereum' +``` +and under ledgers change to: +``` +ledgers: ['ethereum'] +``` + - Generate some wealth to your weather client ETH address: -Go to Metamask [Faucet](https://faucet.metamask.io) and request some test ETH for the account your AEA is using (you need to first load your AEAs private key into MetaMask). +Go to Metamask [Faucet](https://faucet.metamask.io) and request some test ETH for the account your AEA is using (you need to first load your AEAs private key into MetaMask). Your private key is at `weather_client/eth_private_key.txt`. -- Run both agents, as in the previous section. +- Follow the last two stesp from the previous section. diff --git a/packages/skills/weather_client_ledger/behaviours.py b/packages/skills/weather_client_ledger/behaviours.py index eb26265563..3f805c1a0c 100644 --- a/packages/skills/weather_client_ledger/behaviours.py +++ b/packages/skills/weather_client_ledger/behaviours.py @@ -18,10 +18,11 @@ # ------------------------------------------------------------------------------ """This package contains a scaffold of a behaviour.""" -import datetime import logging from typing import cast, TYPE_CHECKING +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.serialization import DEFAULT_OEF, OEFSerializer from aea.skills.base import Behaviour @@ -40,24 +41,24 @@ class MySearchBehaviour(Behaviour): def __init__(self, **kwargs): """Initialise the class.""" super().__init__(**kwargs) - self._search_id = 0 def setup(self) -> None: """Implement the setup for the behaviour.""" - fet_balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) + if self.context.ledger_apis.has_fetchai: + fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + if fet_balance > 0: + logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) + else: + logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) + # TODO: deregister skill from filter - eth_balance = self.context.ledger_apis.token_balance('ethereum', cast(str, self.context.agent_addresses.get('ethereum'))) - if fet_balance > 0: - logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) - else: - logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) - # TODO: deregister skill from filter - - if eth_balance > 0: - logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) - else: - logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) - # TODO: deregister skill from filter + if self.context.ledger_apis.has_ethereum: + eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + if eth_balance > 0: + logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) + else: + logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) + # TODO: deregister skill from filter def act(self) -> None: """ @@ -66,17 +67,16 @@ def act(self) -> None: :return: None """ strategy = cast(Strategy, self.context.strategy) - if strategy.is_searching and strategy.is_time_to_search(): - self._search_id += 1 - strategy.last_search_time = datetime.datetime.now() + if strategy.is_time_to_search(): query = strategy.get_service_query() - search_request = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, - id=self._search_id, - query=query) + search_id = strategy.get_next_search_id() + oef_msg = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, + id=search_id, + query=query) self.context.outbox.put_message(to=DEFAULT_OEF, sender=self.context.agent_public_key, protocol_id=OEFMessage.protocol_id, - message=OEFSerializer().encode(search_request)) + message=OEFSerializer().encode(oef_msg)) def teardown(self) -> None: """ @@ -84,5 +84,10 @@ def teardown(self) -> None: :return: None """ - balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) - logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + if self.context.ledger_apis.has_fetchai: + balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + + if self.context.ledger_apis.has_ethereum: + balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) diff --git a/packages/skills/weather_client_ledger/dialogues.py b/packages/skills/weather_client_ledger/dialogues.py index ffacc494be..bb6b721095 100644 --- a/packages/skills/weather_client_ledger/dialogues.py +++ b/packages/skills/weather_client_ledger/dialogues.py @@ -28,29 +28,18 @@ from enum import Enum import logging -from typing import Any, Dict, Optional, cast +from typing import Dict, Optional, cast from aea.helpers.dialogue.base import DialogueLabel from aea.helpers.dialogue.base import Dialogue as BaseDialogue from aea.mail.base import Address from aea.protocols.base import Message -from aea.protocols.fipa.message import FIPAMessage +from aea.protocols.fipa.message import FIPAMessage, VALID_PREVIOUS_PERFORMATIVES from aea.protocols.oef.models import Description from aea.skills.base import SharedClass -Action = Any logger = logging.getLogger("aea.weather_client_ledger_skill") -STARTING_MESSAGE_ID = 1 -STARTING_MESSAGE_TARGET = 0 -CFP_TARGET = STARTING_MESSAGE_TARGET -PROPOSE_TARGET = CFP_TARGET + 1 -DECLINED_CFP_TARGET = CFP_TARGET + 1 -ACCEPT_TARGET = PROPOSE_TARGET + 1 -MATCH_ACCEPT_TARGET = ACCEPT_TARGET + 1 -DECLINED_ACCEPT_TARGET = ACCEPT_TARGET + 1 -INFORM_TARGET = MATCH_ACCEPT_TARGET + 2 # this INFORM is a response to the own INFORM - class Dialogue(BaseDialogue): """The dialogue class maintains state of a dialogue and manages it.""" @@ -73,54 +62,27 @@ def __init__(self, dialogue_label: DialogueLabel, **kwargs) -> None: BaseDialogue.__init__(self, dialogue_label=dialogue_label) self.proposal = None # type: Optional[Description] - def is_expecting_propose(self) -> bool: - """ - Check whether the dialogue is expecting a propose. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.CFP - return result - - def is_expecting_matching_accept(self) -> bool: - """ - Check whether the dialogue is expecting a matching accept. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.ACCEPT - return result - - def is_expecting_cfp_decline(self) -> bool: - """ - Check whether the dialogue is expecting an decline following a cfp. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.CFP - return result - - def is_expecting_accept_decline(self) -> bool: - """ - Check whether the dialogue is expecting an decline following an accept. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.ACCEPT - return result - - def is_expecting_inform(self) -> bool: + def is_valid_next_message(self, fipa_msg: FIPAMessage) -> bool: """ - Check whether the dialogue is expecting an inform. + Check whether this is a valid next message in the dialogue. :return: True if yes, False otherwise. """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.INFORM + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = cast(FIPAMessage.Performative, fipa_msg.get("performative")) + last_outgoing_message = self.last_outgoing_message + if last_outgoing_message is None: + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP + else: + last_message_id = cast(int, last_outgoing_message.get("message_id")) + last_target = cast(int, last_outgoing_message.get("target")) + last_performative = cast(FIPAMessage.Performative, last_outgoing_message.get("performative")) + result = this_message_id == last_message_id + 1 and \ + this_target == last_target + 1 and \ + last_performative in VALID_PREVIOUS_PERFORMATIVES[this_performative] return result @@ -191,55 +153,28 @@ def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address :return: boolean indicating whether the message belongs to a registered dialogue """ + fipa_msg = cast(FIPAMessage, fipa_msg) dialogue_id = cast(int, fipa_msg.get("dialogue_id")) - target = cast(int, fipa_msg.get("target")) - performative = fipa_msg.get("performative") self_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, agent_pbk) - result = False - if performative == FIPAMessage.Performative.PROPOSE and target == PROPOSE_TARGET and self_initiated_dialogue_label in self.dialogues: - self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - result = self_initiated_dialogue.is_expecting_propose() - elif performative == FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS and target == MATCH_ACCEPT_TARGET and self_initiated_dialogue_label in self.dialogues: - self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - result = self_initiated_dialogue.is_expecting_matching_accept() - elif performative == FIPAMessage.Performative.DECLINE: - if target == DECLINED_CFP_TARGET and self_initiated_dialogue_label in self.dialogues: - self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - result = self_initiated_dialogue.is_expecting_cfp_decline() - elif target == DECLINED_ACCEPT_TARGET and self_initiated_dialogue_label in self.dialogues: - self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - result = self_initiated_dialogue.is_expecting_accept_decline() - elif performative == FIPAMessage.Performative.INFORM and target == INFORM_TARGET and self_initiated_dialogue_label in self.dialogues: + if self_initiated_dialogue_label in self.dialogues: self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - result = self_initiated_dialogue.is_expecting_inform() + result = self_initiated_dialogue.is_valid_next_message(fipa_msg) + else: + result = False return result - def get_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> Dialogue: + def get_dialogue(self, dialogue_id: int, sender: Address, agent_pbk: Address) -> Dialogue: """ Retrieve dialogue. - :param fipa_msg: the fipa message + :param dialogue_id: the dialogue id :param sender_pbk: the sender public key :param agent_pbk: the public key of the agent :return: the dialogue """ - dialogue_id = cast(int, fipa_msg.get("dialogue_id")) - target = cast(int, fipa_msg.get("target")) - performative = fipa_msg.get("performative") self_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, agent_pbk) - if (performative == FIPAMessage.Performative.PROPOSE and target == PROPOSE_TARGET and self_initiated_dialogue_label in self.dialogues) \ - or \ - (performative == FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS and target == MATCH_ACCEPT_TARGET and self_initiated_dialogue_label in self.dialogues) \ - or \ - (performative == FIPAMessage.Performative.DECLINE and target == DECLINED_CFP_TARGET and self_initiated_dialogue_label in self.dialogues) \ - or \ - (performative == FIPAMessage.Performative.DECLINE and target == DECLINED_ACCEPT_TARGET and self_initiated_dialogue_label in self.dialogues) \ - or \ - (performative == FIPAMessage.Performative.INFORM and target == INFORM_TARGET and self_initiated_dialogue_label in self.dialogues): - dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) - else: - raise ValueError('Should have found dialogue.') + dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) return dialogue def create_self_initiated(self, dialogue_opponent_pbk: Address, dialogue_starter_pbk: Address) -> Dialogue: diff --git a/packages/skills/weather_client_ledger/handlers.py b/packages/skills/weather_client_ledger/handlers.py index 4815c3db88..b20f0abdb1 100644 --- a/packages/skills/weather_client_ledger/handlers.py +++ b/packages/skills/weather_client_ledger/handlers.py @@ -43,20 +43,12 @@ logger = logging.getLogger("aea.weather_client_ledger_skill") -STARTING_MESSAGE_ID = 1 -STARTING_TARGET_ID = 0 -DEFAULT_MAX_PRICE = 2.0 - class FIPAHandler(Handler): """This class scaffolds a handler.""" SUPPORTED_PROTOCOL = FIPAMessage.protocol_id # type: Optional[ProtocolId] - def __init__(self, **kwargs): - """Initialise the class.""" - super().__init__(**kwargs) - def setup(self) -> None: """ Implement the setup. @@ -82,7 +74,7 @@ def handle(self, message: Message, sender: str) -> None: # recover dialogue dialogues = cast(Dialogues, self.context.dialogues) if dialogues.is_belonging_to_registered_dialogue(fipa_msg, sender, self.context.agent_public_key): - dialogue = dialogues.get_dialogue(fipa_msg, sender, self.context.agent_public_key) + dialogue = dialogues.get_dialogue(dialogue_id, sender, self.context.agent_public_key) dialogue.incoming_extend(fipa_msg) else: self._handle_unidentified_dialogue(fipa_msg, sender) @@ -117,7 +109,7 @@ def _handle_unidentified_dialogue(self, msg: FIPAMessage, sender: str) -> None: default_msg = DefaultMessage(type=DefaultMessage.Type.ERROR, error_code=DefaultMessage.ErrorCode.INVALID_DIALOGUE.value, error_msg="Invalid dialogue.", - error_data="fipa_message") # FIPASerializer().encode(msg)) + error_data="fipa_message") # TODO: send back FIPASerializer().encode(msg)) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, protocol_id=DefaultMessage.protocol_id, @@ -141,14 +133,12 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog if proposals is not []: # only take the first proposal proposal = proposals[0] - ledger_id = cast(str, proposal.values.get('ledger_id')) logger.info("[{}]: received proposal={} from sender={}".format(self.context.agent_name, proposal.values, sender[-5:])) strategy = cast(Strategy, self.context.strategy) acceptable = strategy.is_acceptable_proposal(proposal) - affordable = self.context.ledger_apis.token_balance(ledger_id, cast(str, self.context.agent_addresses.get( - ledger_id))) >= cast(int, proposal.values.get('price')) + affordable = strategy.is_affordable_proposal(proposal) if acceptable and affordable: strategy.is_searching = False logger.info("[{}]: accepting the proposal from sender={}".format(self.context.agent_name, @@ -209,6 +199,7 @@ def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, d """ logger.info("[{}]: received MATCH_ACCEPT_W_ADDRESS from sender={}".format(self.context.agent_name, sender[-5:])) address = cast(str, msg.get("address")) + strategy = cast(Strategy, self.context.strategy) proposal = cast(Description, dialogue.proposal) ledger_id = cast(str, proposal.values.get("ledger_id")) tx_msg = TransactionMessage(performative=TransactionMessage.Performative.PROPOSE, @@ -217,10 +208,10 @@ def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, d sender=self.context.agent_public_keys[ledger_id], counterparty=address, is_sender_buyer=True, - currency_pbk=cast(str, proposal.values.get("currency_pbk")), + currency_pbk=proposal.values['currency_pbk'], amount=proposal.values['price'], - sender_tx_fee=2000000, # TODO to be read from configurations - counterparty_tx_fee=0, + sender_tx_fee=strategy.max_buyer_tx_fee, + counterparty_tx_fee=proposal.values['seller_tx_fee'], quantities_by_good_pbk={}, dialogue_label=dialogue.dialogue_label.json, ledger_id=ledger_id) @@ -257,10 +248,6 @@ class OEFHandler(Handler): SUPPORTED_PROTOCOL = OEFMessage.protocol_id # type: Optional[ProtocolId] - def __init__(self, **kwargs): - """Initialise the oef handler.""" - super().__init__(**kwargs) - def setup(self) -> None: """Call to setup the handler.""" pass @@ -307,10 +294,10 @@ def _handle_search(self, agents: List[str]) -> None: dialogue = dialogues.create_self_initiated(opponent_pbk, self.context.agent_public_key) query = strategy.get_service_query() logger.info("[{}]: sending CFP to agent={}".format(self.context.agent_name, opponent_pbk[-5:])) - cfp_msg = FIPAMessage(message_id=STARTING_MESSAGE_ID, + cfp_msg = FIPAMessage(message_id=FIPAMessage.STARTING_MESSAGE_ID, dialogue_id=dialogue.dialogue_label.dialogue_id, performative=FIPAMessage.Performative.CFP, - target=STARTING_TARGET_ID, + target=FIPAMessage.STARTING_TARGET, query=query) dialogue.outgoing_extend(cfp_msg) self.context.outbox.put_message(to=opponent_pbk, diff --git a/packages/skills/weather_client_ledger/skill.yaml b/packages/skills/weather_client_ledger/skill.yaml index 8e3ef5b420..88f15b8b34 100644 --- a/packages/skills/weather_client_ledger/skill.yaml +++ b/packages/skills/weather_client_ledger/skill.yaml @@ -24,7 +24,10 @@ shared_classes: args: country: UK search_interval: 5 - max_price: 50 + max_row_price: 4 + max_buyer_tx_fee: 1 + currency_pbk: 'FET' + ledger_id: 'fetchai' - shared_class: class_name: Dialogues args: {} diff --git a/packages/skills/weather_client_ledger/strategy.py b/packages/skills/weather_client_ledger/strategy.py index a0b3e86eb7..60fa0f3279 100644 --- a/packages/skills/weather_client_ledger/strategy.py +++ b/packages/skills/weather_client_ledger/strategy.py @@ -28,7 +28,10 @@ DEFAULT_COUNTRY = 'UK' SEARCH_TERM = 'country' DEFAULT_SEARCH_INTERVAL = 5.0 -DEFAULT_MAX_PRICE = 5 +DEFAULT_MAX_ROW_PRICE = 5 +DEFAULT_MAX_TX_FEE = 2 +DEFAULT_CURRENCY_PBK = 'FET' +DEFAULT_LEDGER_ID = 'fetchai' class Strategy(SharedClass): @@ -42,10 +45,24 @@ def __init__(self, **kwargs) -> None: """ self._country = kwargs.pop('country') if 'country' in kwargs.keys() else DEFAULT_COUNTRY self._search_interval = cast(float, kwargs.pop('search_interval')) if 'search_interval' in kwargs.keys() else DEFAULT_SEARCH_INTERVAL - self._max_price = kwargs.pop('max_price') if 'max_price' in kwargs.keys() else DEFAULT_MAX_PRICE + self._max_row_price = kwargs.pop('max_row_price') if 'max_row_price' in kwargs.keys() else DEFAULT_MAX_ROW_PRICE + self.max_buyer_tx_fee = kwargs.pop('max_tx_fee') if 'max_tx_fee' in kwargs.keys() else DEFAULT_MAX_TX_FEE + self._currency_pbk = kwargs.pop('currency_pbk') if 'currency_pbk' in kwargs.keys() else DEFAULT_CURRENCY_PBK + self._ledger_id = kwargs.pop('ledger_id') if 'ledger_id' in kwargs.keys() else DEFAULT_LEDGER_ID super().__init__(**kwargs) + self._search_id = 0 self.is_searching = True - self.last_search_time = datetime.datetime.now() + self._last_search_time = datetime.datetime.now() + + def get_next_search_id(self) -> int: + """ + Get the next search id and set the search time. + + :return: the next search id + """ + self._search_id += 1 + self._last_search_time = datetime.datetime.now() + return self._search_id def get_service_query(self) -> Query: """ @@ -62,8 +79,10 @@ def is_time_to_search(self) -> bool: :return: whether it is time to search """ + if not self.is_searching: + return False now = datetime.datetime.now() - diff = now - self.last_search_time + diff = now - self._last_search_time result = diff.total_seconds() > self._search_interval return result @@ -73,5 +92,20 @@ def is_acceptable_proposal(self, proposal: Description) -> bool: :return: whether it is acceptable """ - result = True if proposal.values["price"] < self._max_price * proposal.values['rows'] else False + result = (proposal.values['price'] - proposal.values['seller_tx_fee'] > 0) and \ + (proposal.values['price'] <= self._max_row_price * proposal.values['rows']) and \ + (proposal.values['currency_pbk'] == self._currency_pbk) and \ + (proposal.values['ledger_id'] == self._ledger_id) return result + + def is_affordable_proposal(self, proposal: Description) -> bool: + """ + Check whether it is an affordable proposal. + + :return: whether it is affordable + """ + payable = proposal.values['price'] + self.max_buyer_tx_fee + ledger_id = proposal.values['ledger_id'] + address = cast(str, self.context.agent_addresses.get(ledger_id)) + balance = self.context.ledger_apis.token_balance(ledger_id, address) + return balance >= payable diff --git a/packages/skills/weather_station_ledger/behaviours.py b/packages/skills/weather_station_ledger/behaviours.py index 266f41eae6..b61897e649 100644 --- a/packages/skills/weather_station_ledger/behaviours.py +++ b/packages/skills/weather_station_ledger/behaviours.py @@ -22,6 +22,8 @@ import logging from typing import cast, TYPE_CHECKING +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI from aea.skills.base import Behaviour from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.serialization import OEFSerializer, DEFAULT_OEF @@ -33,8 +35,6 @@ logger = logging.getLogger("aea.weather_station_ledger_skill") -REGISTER_ID = 1 -UNREGISTER_ID = 2 SERVICE_ID = '' @@ -52,15 +52,26 @@ def setup(self) -> None: :return: None """ - balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) - logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, balance)) - balance = self.context.ledger_apis.token_balance('ethereum', cast(str, self.context.agent_addresses.get('ethereum'))) - logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, balance)) + if self.context.ledger_apis.has_fetchai: + fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + if fet_balance > 0: + logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) + else: + logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) + + if self.context.ledger_apis.has_ethereum: + eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + if eth_balance > 0: + logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) + else: + logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) + if not self._registered: strategy = cast(Strategy, self.context.strategy) desc = strategy.get_service_description() + oef_msg_id = strategy.get_next_oef_msg_id() msg = OEFMessage(oef_type=OEFMessage.Type.REGISTER_SERVICE, - id=REGISTER_ID, + id=oef_msg_id, service_description=desc, service_id=SERVICE_ID) self.context.outbox.put_message(to=DEFAULT_OEF, @@ -84,13 +95,20 @@ def teardown(self) -> None: :return: None """ - balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) - logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + if self.context.ledger_apis.has_fetchai: + balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + + if self.context.ledger_apis.has_ethereum: + balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) + if self._registered: strategy = cast(Strategy, self.context.strategy) desc = strategy.get_service_description() + oef_msg_id = strategy.get_next_oef_msg_id() msg = OEFMessage(oef_type=OEFMessage.Type.UNREGISTER_SERVICE, - id=UNREGISTER_ID, + id=oef_msg_id, service_description=desc, service_id=SERVICE_ID) self.context.outbox.put_message(to=DEFAULT_OEF, diff --git a/packages/skills/weather_station_ledger/dialogues.py b/packages/skills/weather_station_ledger/dialogues.py index 74d4175cfe..f844088610 100644 --- a/packages/skills/weather_station_ledger/dialogues.py +++ b/packages/skills/weather_station_ledger/dialogues.py @@ -34,22 +34,12 @@ from aea.helpers.dialogue.base import Dialogue as BaseDialogue from aea.mail.base import Address from aea.protocols.base import Message -from aea.protocols.fipa.message import FIPAMessage +from aea.protocols.fipa.message import FIPAMessage, VALID_PREVIOUS_PERFORMATIVES from aea.protocols.oef.models import Description from aea.skills.base import SharedClass -Action = Any logger = logging.getLogger("aea.weather_station_ledger_skill") -STARTING_MESSAGE_ID = 1 -STARTING_MESSAGE_TARGET = 0 -CFP_TARGET = STARTING_MESSAGE_TARGET -PROPOSE_TARGET = CFP_TARGET + 1 -ACCEPT_TARGET = PROPOSE_TARGET + 1 -DECLINED_PROPOSE_TARGET = PROPOSE_TARGET + 1 -MATCH_ACCEPT_TARGET = ACCEPT_TARGET + 1 -INFORM_TARGET = MATCH_ACCEPT_TARGET + 1 - class Dialogue(BaseDialogue): """The dialogue class maintains state of a dialogue and manages it.""" @@ -72,34 +62,27 @@ def __init__(self, dialogue_label: DialogueLabel, **kwargs) -> None: self.weather_data = None # type: Optional[Dict[str, Any]] self.proposal = None # type: Optional[Description] - def is_expecting_accept(self) -> bool: + def is_valid_next_message(self, fipa_msg: FIPAMessage) -> bool: """ - Check whether the dialogue is expecting an initial accept. + Check whether this is a valid next message in the dialogue. :return: True if yes, False otherwise. """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.PROPOSE - return result - - def is_expecting_propose_decline(self) -> bool: - """ - Check whether the dialogue is expecting an decline following a propose. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.PROPOSE - return result - - def is_expecting_inform(self) -> bool: - """ - Check whether the dialogue is expecting an inform. - - :return: True if yes, False otherwise. - """ - last_sent_message = self._outgoing_messages[-1] if len(self._outgoing_messages) > 0 else None # type: Optional[Message] - result = (last_sent_message is not None) and last_sent_message.get("performative") == FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = cast(FIPAMessage.Performative, fipa_msg.get("performative")) + last_outgoing_message = self.last_outgoing_message + if last_outgoing_message is None: + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP + else: + last_message_id = cast(int, last_outgoing_message.get("message_id")) + last_target = cast(int, last_outgoing_message.get("target")) + last_performative = cast(FIPAMessage.Performative, last_outgoing_message.get("performative")) + result = this_message_id == last_message_id + 1 and \ + this_target == last_target + 1 and \ + last_performative in VALID_PREVIOUS_PERFORMATIVES[this_performative] return result @@ -164,13 +147,13 @@ def is_permitted_for_new_dialogue(self, fipa_msg: Message, sender: Address) -> b :return: a boolean indicating whether the message is permitted for a new dialogue """ fipa_msg = cast(FIPAMessage, fipa_msg) - msg_id = fipa_msg.get("message_id") - target = fipa_msg.get("target") - performative = fipa_msg.get("performative") + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = fipa_msg.get("performative") - result = performative == FIPAMessage.Performative.CFP \ - and msg_id == STARTING_MESSAGE_ID \ - and target == CFP_TARGET + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP return result def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> bool: @@ -183,69 +166,41 @@ def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address :return: boolean indicating whether the message belongs to a registered dialogue """ + fipa_msg = cast(FIPAMessage, fipa_msg) dialogue_id = cast(int, fipa_msg.get("dialogue_id")) - target = cast(int, fipa_msg.get("target")) - performative = fipa_msg.get("performative") other_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, sender) - result = False - if performative == FIPAMessage.Performative.ACCEPT: - if target == ACCEPT_TARGET and other_initiated_dialogue_label in self.dialogues: - other_initiated_dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) - result = other_initiated_dialogue.is_expecting_accept() - elif performative == FIPAMessage.Performative.DECLINE: - if target == DECLINED_PROPOSE_TARGET and other_initiated_dialogue_label in self.dialogues: - other_initiated_dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) - result = other_initiated_dialogue.is_expecting_propose_decline() - elif performative == FIPAMessage.Performative.INFORM: - if target == INFORM_TARGET and other_initiated_dialogue_label in self.dialogues: - other_initiated_dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) - result = other_initiated_dialogue.is_expecting_inform() + if other_initiated_dialogue_label in self.dialogues: + other_initiated_dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) + result = other_initiated_dialogue.is_valid_next_message(fipa_msg) + else: + result = False return result - def get_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> Dialogue: + def get_dialogue(self, dialogue_id: int, sender: Address, agent_pbk: Address) -> Dialogue: """ Retrieve dialogue. - :param fipa_msg: the fipa message + :param dialogue_id: the dialogue id :param sender_pbk: the sender public key :param agent_pbk: the public key of the agent :return: the dialogue """ - dialogue_id = cast(int, fipa_msg.get("dialogue_id")) - target = cast(int, fipa_msg.get("target")) - performative = fipa_msg.get("performative") other_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, sender) - if performative == FIPAMessage.Performative.ACCEPT: - if target == ACCEPT_TARGET and other_initiated_dialogue_label in self.dialogues: - dialogue = self.dialogues[other_initiated_dialogue_label] - else: - raise ValueError('Should have found dialogue.') - elif performative == FIPAMessage.Performative.DECLINE: - if target == DECLINED_PROPOSE_TARGET and other_initiated_dialogue_label in self.dialogues: - dialogue = self.dialogues[other_initiated_dialogue_label] - else: - raise ValueError('Should have found dialogue.') - elif performative == FIPAMessage.Performative.INFORM: - if target == INFORM_TARGET and other_initiated_dialogue_label in self.dialogues: - dialogue = self.dialogues[other_initiated_dialogue_label] - else: - raise ValueError('Should have found dialogue.') - dialogue = cast(Dialogue, dialogue) + dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) return dialogue - def create_opponent_initiated(self, fipa_msg: FIPAMessage, sender: Address) -> Dialogue: + def create_opponent_initiated(self, dialogue_id: int, sender: Address) -> Dialogue: """ Save an opponent initiated dialogue. - :param fipa_msg: the fipa message + :param dialogue_id: the dialogue id :param sender: the pbk of the sender :return: the created dialogue """ dialogue_starter_pbk = sender dialogue_opponent_pbk = sender - dialogue_id = cast(int, fipa_msg.get("dialogue_id")) dialogue_label = DialogueLabel(dialogue_id, dialogue_opponent_pbk, dialogue_starter_pbk) result = self._create(dialogue_label) return result diff --git a/packages/skills/weather_station_ledger/handlers.py b/packages/skills/weather_station_ledger/handlers.py index ee248c4d14..27c85ecdc5 100644 --- a/packages/skills/weather_station_ledger/handlers.py +++ b/packages/skills/weather_station_ledger/handlers.py @@ -67,10 +67,10 @@ def handle(self, message: Message, sender: str) -> None: # recover dialogue dialogues = cast(Dialogues, self.context.dialogues) if dialogues.is_belonging_to_registered_dialogue(fipa_msg, sender, self.context.agent_public_key): - dialogue = dialogues.get_dialogue(fipa_msg, sender, self.context.agent_public_key) + dialogue = dialogues.get_dialogue(dialogue_id, sender, self.context.agent_public_key) dialogue.incoming_extend(fipa_msg) elif dialogues.is_permitted_for_new_dialogue(fipa_msg, sender): - dialogue = dialogues.create_opponent_initiated(fipa_msg, sender) + dialogue = dialogues.create_opponent_initiated(dialogue_id, sender) dialogue.incoming_extend(fipa_msg) else: self._handle_unidentified_dialogue(fipa_msg, sender) diff --git a/packages/skills/weather_station_ledger/skill.yaml b/packages/skills/weather_station_ledger/skill.yaml index be450a65c0..cd1dcbec6e 100644 --- a/packages/skills/weather_station_ledger/skill.yaml +++ b/packages/skills/weather_station_ledger/skill.yaml @@ -16,11 +16,12 @@ shared_classes: - shared_class: class_name: Strategy args: + date_one: "1/10/2019" + date_two: "1/12/2019" price_per_row: 1 + seller_tx_fee: 0 currency_pbk: 'FET' ledger_id: 'fetchai' - # currency_pbk: 'ETH' - # ledger_id: 'ethereum' - shared_class: class_name: Dialogues args: {} diff --git a/packages/skills/weather_station_ledger/strategy.py b/packages/skills/weather_station_ledger/strategy.py index 35d0318547..91f64ce89f 100644 --- a/packages/skills/weather_station_ledger/strategy.py +++ b/packages/skills/weather_station_ledger/strategy.py @@ -32,10 +32,11 @@ from weather_station_ledger_skill.weather_station_data_model import WEATHER_STATION_DATAMODEL, SCHEME DEFAULT_PRICE_PER_ROW = 2 +DEFAULT_SELLER_TX_FEE = 0 DEFAULT_CURRENCY_PBK = 'FET' DEFAULT_LEDGER_ID = 'fetchai' -DATE_ONE = "3/10/2019" -DATE_TWO = "15/10/2019" +DEFAULT_DATE_ONE = "3/10/2019" +DEFAULT_DATE_TWO = "15/10/2019" class Strategy(SharedClass): @@ -50,11 +51,24 @@ def __init__(self, **kwargs) -> None: :return: None """ - self.price_per_row = kwargs.pop('price_per_row') if 'price_per_row' in kwargs.keys() else DEFAULT_PRICE_PER_ROW - self.currency_pbk = kwargs.pop('currency_pbk') if 'currency_pbk' in kwargs.keys() else DEFAULT_CURRENCY_PBK - self.ledger_id = kwargs.pop('ledger_id') if 'ledger_id' in kwargs.keys() else DEFAULT_LEDGER_ID + self._price_per_row = kwargs.pop('price_per_row') if 'price_per_row' in kwargs.keys() else DEFAULT_PRICE_PER_ROW + self._seller_tx_fee = kwargs.pop('seller_tx_fee') if 'seller_tx_fee' in kwargs.keys() else DEFAULT_SELLER_TX_FEE + self._currency_pbk = kwargs.pop('currency_pbk') if 'currency_pbk' in kwargs.keys() else DEFAULT_CURRENCY_PBK + self._ledger_id = kwargs.pop('ledger_id') if 'ledger_id' in kwargs.keys() else DEFAULT_LEDGER_ID + self._date_one = kwargs.pop('date_one') if 'date_one' in kwargs.keys() else DEFAULT_DATE_ONE + self._date_two = kwargs.pop('date_two') if 'date_two' in kwargs.keys() else DEFAULT_DATE_TWO super().__init__(**kwargs) self.db = DBCommunication() + self._oef_msg_id = 0 + + def get_next_oef_msg_id(self) -> int: + """ + Get the next oef msg id. + + :return: the next oef msg id + """ + self._oef_msg_id += 1 + return self._oef_msg_id def get_service_description(self) -> Description: """ @@ -82,14 +96,15 @@ def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[st :param query: the query :return: a tuple of proposal and the weather data """ - # TODO, this is a stub - fetched_data = self.db.get_data_for_specific_dates(DATE_ONE, DATE_TWO) + fetched_data = self.db.get_data_for_specific_dates(self._date_one, self._date_two) # TODO: fetch real data weather_data, rows = self._build_data_payload(fetched_data) - total_price = self.price_per_row * rows + total_price = self._price_per_row * rows + assert total_price - self._seller_tx_fee > 0, "This sale would generate a loss, change the configs!" proposal = Description({"rows": rows, "price": total_price, - "currency_pbk": self.currency_pbk, - "ledger_id": self.ledger_id}) + "seller_tx_fee": self._seller_tx_fee, + "currency_pbk": self._currency_pbk, + "ledger_id": self._ledger_id}) return (proposal, weather_data) def _build_data_payload(self, fetched_data: Dict[str, int]) -> Tuple[Dict[str, List[Dict[str, Any]]], int]: From c486b506acda37de385071bca34bd89d74d2d5b3 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 09:28:51 +0000 Subject: [PATCH 11/38] crypt_tests WIP --- aea/crypto/helpers.py | 17 ++++++++--------- tests/data/dummy_aea/default_private_key.pem | 8 ++++---- tests/data/dummy_aea/eth_private_key.txt | 2 +- tests/data/dummy_aea/fet_private_key.txt | 2 +- tests/test_crypto/test_helpers.py | 7 ++++++- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index dbada99ae0..464bb1c0b8 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -19,7 +19,6 @@ # ------------------------------------------------------------------------------ """Module wrapping the helpers of public and private key cryptography.""" -import sys from typing import cast from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption @@ -70,7 +69,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_private_key_pem_path(default_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(default_private_key_config.path), default_private_key_config.ledger)) - sys.exit(1) + exit(-1) fetchai_private_key_config = aea_conf.private_key_paths.read(FETCHAI) if fetchai_private_key_config is None: @@ -87,7 +86,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_fet_private_key_path(fetchai_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(fetchai_private_key_config.path), fetchai_private_key_config.ledger)) - sys.exit(1) + exit(-1) ethereum_private_key_config = aea_conf.private_key_paths.read(ETHEREUM) if ethereum_private_key_config is None: @@ -104,7 +103,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_ethereum_private_key_path(ethereum_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(ethereum_private_key_config.path), ethereum_private_key_config.ledger)) - sys.exit(1) + exit(-1) # update aea config path = Path(DEFAULT_AEA_CONFIG_FILE) @@ -138,7 +137,7 @@ def _verify_ledger_apis_access(ctx: Context) -> None: LedgerApi(fetchai_ledger_api_config.addr, fetchai_ledger_api_config.port) except Exception: logger.error("Cannot connect to fetchai ledger with provided config.") - sys.exit(1) + exit(-1) ethereum_ledger_config = aea_conf.ledger_apis.read(ETHEREUM) if ethereum_ledger_config is None: @@ -150,7 +149,7 @@ def _verify_ledger_apis_access(ctx: Context) -> None: Web3(HTTPProvider(ethereum_ledger_config.addr)) except Exception: logger.error("Cannot connect to ethereum ledger with provided config.") - sys.exit(1) + exit(-1) def _create_temporary_private_key() -> bytes: @@ -176,7 +175,7 @@ def _try_validate_private_key_pem_path(private_key_pem_path: str) -> None: DefaultCrypto(private_key_pem_path=private_key_pem_path) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_pem_path)) - sys.exit(1) + exit(-1) def _try_validate_fet_private_key_path(private_key_path: str) -> None: @@ -193,7 +192,7 @@ def _try_validate_fet_private_key_path(private_key_path: str) -> None: Entity.from_hex(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - sys.exit(1) + exit(-1) def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: @@ -210,7 +209,7 @@ def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: Account.from_key(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - sys.exit(1) + exit(-1) def _create_temporary_private_key_pem_path() -> str: diff --git a/tests/data/dummy_aea/default_private_key.pem b/tests/data/dummy_aea/default_private_key.pem index 344cc23d2c..45cbcf576e 100644 --- a/tests/data/dummy_aea/default_private_key.pem +++ b/tests/data/dummy_aea/default_private_key.pem @@ -1,6 +1,6 @@ -----BEGIN EC PRIVATE KEY----- -MIGkAgEBBDAvooixCAY5kO+PORV2oblpbKkPQ6kG41J6PrJvYULOGCltjNnDu7nT -rhpKLSL4zmKgBwYFK4EEACKhZANiAATje8YjXsjyNbOcqsfSGKf7dqncNZ43j79M -Cj0Ez52VcunGktL0mUqa+fVaN9LD+T5TyfyiViw1FzVHTPmqlp6kZVYrH/zJDbVw -dsdooWy3LOfhf8hak4XORcLcdUa22ys= +MIGkAgEBBDBPDMr3mGOklmP20XAcuJWXyi7MrqEpXnIpLMSrlxRfCt+xToUULRuc +13ZfEf6/h+ygBwYFK4EEACKhZANiAASCAxxhmfAN7IU/7TBnmadwFJzNYuIcBCZW +0vyazEyxuZCR0PeSJELVNNr0XCjV65ph+2g48rv/RvrBLC60fglCOVBkZcccWSLD +S6yukJFBG+z27TE3+O4M0HwC83mLKFc= -----END EC PRIVATE KEY----- diff --git a/tests/data/dummy_aea/eth_private_key.txt b/tests/data/dummy_aea/eth_private_key.txt index 43bf869846..09064f89fa 100644 --- a/tests/data/dummy_aea/eth_private_key.txt +++ b/tests/data/dummy_aea/eth_private_key.txt @@ -1 +1 @@ -0xf5c605c8a611aed64099548aae758fcb7cc364cf896236efe3d478a16cfd6b63 \ No newline at end of file +0x6F611408F7EF304947621C51A4B7D84A13A2B9786E9F984DA790A096E8260C64 \ No newline at end of file diff --git a/tests/data/dummy_aea/fet_private_key.txt b/tests/data/dummy_aea/fet_private_key.txt index e23683c0eb..af8f967be5 100644 --- a/tests/data/dummy_aea/fet_private_key.txt +++ b/tests/data/dummy_aea/fet_private_key.txt @@ -1 +1 @@ -3186c61cbd181fcefe65ff836128d8c9511305dd8ff568d999d911a25ce602ec \ No newline at end of file +66cec3f67a5fa81b6eb1c3c678dd60bb6959e3930c452397196bd63b45af5f00 \ No newline at end of file diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index 51a2ca0abb..02d20d0dcd 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -19,14 +19,19 @@ # ------------------------------------------------------------------------------ """This module contains the tests for the crypto/helpers module.""" +import logging + import os import pytest - from aea.crypto.helpers import _try_validate_private_key_pem_path, _try_validate_fet_private_key_path, \ _try_validate_ethereum_private_key_path + from tests.conftest import CUR_PATH +logger = logging.getLogger(__name__) + + class TestHelperFile: """Test helper module in aea/crypto.""" From 2aa9662830ac1d69ab1daafc40599751820f2750 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 09:41:13 +0000 Subject: [PATCH 12/38] Revert the amount in the settle_transaction --- aea/crypto/ledger_apis.py | 2 +- packages/skills/carpark_detection/handlers.py | 8 ++++---- packages/skills/weather_station_ledger/handlers.py | 4 ++-- tests/test_crypto/test_ledger_apis.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/aea/crypto/ledger_apis.py b/aea/crypto/ledger_apis.py index 6d7efa822e..cf9af554f7 100644 --- a/aea/crypto/ledger_apis.py +++ b/aea/crypto/ledger_apis.py @@ -159,7 +159,7 @@ def transfer(self, identifier: str, crypto_object: Crypto, destination_address: tx_digest = None return tx_digest - def is_tx_settled(self, identifier: str, tx_digest: str) -> bool: + def is_tx_settled(self, identifier: str, tx_digest: str, amount: int) -> bool: """ Check whether the transaction is settled and correct. diff --git a/packages/skills/carpark_detection/handlers.py b/packages/skills/carpark_detection/handlers.py index 64a8db8773..5012c18287 100644 --- a/packages/skills/carpark_detection/handlers.py +++ b/packages/skills/carpark_detection/handlers.py @@ -28,7 +28,7 @@ from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer -from aea.protocols.oef.models import Query # Description +from aea.protocols.oef.models import Query, Description from aea.skills.base import Handler if TYPE_CHECKING: @@ -236,9 +236,9 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu tx_digest = json_data['transaction_digest'] logger.info("[{}]: checking whether transaction={} has been received ...".format(self.context.agent_name, tx_digest)) - # proposal = cast(Description, dialogue.proposal) - # total_price = cast(int, proposal.values.get("price")) - is_settled = self.context.ledger_apis.is_tx_settled('fetchai', tx_digest) + proposal = cast(Description, dialogue.proposal) + total_price = cast(int, proposal.values.get("price")) + is_settled = self.context.ledger_apis.is_tx_settled('fetchai', tx_digest, total_price) if is_settled: token_balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) diff --git a/packages/skills/weather_station_ledger/handlers.py b/packages/skills/weather_station_ledger/handlers.py index 603402c6e7..ee248c4d14 100644 --- a/packages/skills/weather_station_ledger/handlers.py +++ b/packages/skills/weather_station_ledger/handlers.py @@ -239,9 +239,9 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu logger.info("[{}]: checking whether transaction={} has been received ...".format(self.context.agent_name, tx_digest)) proposal = cast(Description, dialogue.proposal) - # total_price = cast(int, proposal.values.get("price")) + total_price = cast(int, proposal.values.get("price")) ledger_id = cast(str, proposal.values.get("ledger_id")) - is_settled = self.context.ledger_apis.is_tx_settled(ledger_id, tx_digest) + is_settled = self.context.ledger_apis.is_tx_settled(ledger_id, tx_digest, total_price) if is_settled: token_balance = self.context.ledger_apis.token_balance(ledger_id, cast(str, self.context.agent_addresses.get(ledger_id))) diff --git a/tests/test_crypto/test_ledger_apis.py b/tests/test_crypto/test_ledger_apis.py index b238604044..58dff81082 100644 --- a/tests/test_crypto/test_ledger_apis.py +++ b/tests/test_crypto/test_ledger_apis.py @@ -72,14 +72,14 @@ def test_transfer(self): FETCHAI: DEFAULT_FETCHAI_CONFIG}) tx_digest = ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) assert tx_digest is not None - assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest) + assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) tx_digest = "unknown_hash" - assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest) + assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=10, tx_fee=200000) assert tx_digest is not None - assert ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest) + assert ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest, amount=10) tx_digest = "unknown_hash" - assert not ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest) + assert not ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest, amount=10) with pytest.raises(ValueError): tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=50, tx_fee=2) From d584146dc552c0bf1a513a61c1f62aa98e637958 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 10:01:35 +0000 Subject: [PATCH 13/38] Revert sys.exit --- aea/crypto/helpers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index 464bb1c0b8..ab15277d64 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -19,6 +19,7 @@ # ------------------------------------------------------------------------------ """Module wrapping the helpers of public and private key cryptography.""" +import sys from typing import cast from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption @@ -69,7 +70,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_private_key_pem_path(default_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(default_private_key_config.path), default_private_key_config.ledger)) - exit(-1) + sys.exit(-1) fetchai_private_key_config = aea_conf.private_key_paths.read(FETCHAI) if fetchai_private_key_config is None: @@ -86,7 +87,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_fet_private_key_path(fetchai_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(fetchai_private_key_config.path), fetchai_private_key_config.ledger)) - exit(-1) + sys.exit(-1) ethereum_private_key_config = aea_conf.private_key_paths.read(ETHEREUM) if ethereum_private_key_config is None: @@ -103,7 +104,7 @@ def _verify_or_create_private_keys(ctx: Context) -> None: _try_validate_ethereum_private_key_path(ethereum_private_key_config.path) except FileNotFoundError: logger.error("File {} for private key {} not found.".format(repr(ethereum_private_key_config.path), ethereum_private_key_config.ledger)) - exit(-1) + sys.exit(-1) # update aea config path = Path(DEFAULT_AEA_CONFIG_FILE) @@ -137,7 +138,7 @@ def _verify_ledger_apis_access(ctx: Context) -> None: LedgerApi(fetchai_ledger_api_config.addr, fetchai_ledger_api_config.port) except Exception: logger.error("Cannot connect to fetchai ledger with provided config.") - exit(-1) + sys.exit(-1) ethereum_ledger_config = aea_conf.ledger_apis.read(ETHEREUM) if ethereum_ledger_config is None: @@ -149,7 +150,7 @@ def _verify_ledger_apis_access(ctx: Context) -> None: Web3(HTTPProvider(ethereum_ledger_config.addr)) except Exception: logger.error("Cannot connect to ethereum ledger with provided config.") - exit(-1) + sys.exit(-1) def _create_temporary_private_key() -> bytes: @@ -175,7 +176,7 @@ def _try_validate_private_key_pem_path(private_key_pem_path: str) -> None: DefaultCrypto(private_key_pem_path=private_key_pem_path) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_pem_path)) - exit(-1) + sys.exit(-1) def _try_validate_fet_private_key_path(private_key_path: str) -> None: @@ -192,7 +193,7 @@ def _try_validate_fet_private_key_path(private_key_path: str) -> None: Entity.from_hex(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - exit(-1) + sys.exit(-1) def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: @@ -209,7 +210,7 @@ def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: Account.from_key(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - exit(-1) + sys.exit(-1) def _create_temporary_private_key_pem_path() -> str: From 8bd4cd81bcc18e3a2c719967de0bf06d47195c59 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Nov 2019 12:08:07 +0000 Subject: [PATCH 14/38] Update docs and minor fixes --- docs/app-areas.md | 2 +- docs/core-components.md | 4 +- docs/index.md | 28 ++--- docs/oef-ledger.md | 2 +- docs/quickstart.md | 68 +++++------- docs/trust.md | 10 +- docs/weather-skills.md | 175 ++++++++++++++++++++++++------ examples/echo_skill/README.md | 43 -------- examples/gym_ex/proxy/agent.py | 2 +- examples/gym_skill/README.md | 39 ------- examples/weather_skills/README.md | 120 -------------------- mkdocs.yml | 8 +- 12 files changed, 188 insertions(+), 313 deletions(-) delete mode 100644 examples/echo_skill/README.md delete mode 100644 examples/gym_skill/README.md delete mode 100644 examples/weather_skills/README.md diff --git a/docs/app-areas.md b/docs/app-areas.md index 64a0b3ae26..ee9ba036f1 100644 --- a/docs/app-areas.md +++ b/docs/app-areas.md @@ -1,6 +1,6 @@ An autonomous economic agent (AEA) is an intelligent agent whose goal is generating economic value for its owner. It can represent machines, humans, or data. -There are five general application areas for Fetch.ai agents. +There are five general application areas for Fetch.ai AEAs. * **Inhabitants**: agents paired with real world hardware devices such as drones, laptops, heat sensors, etc. * **Interfaces**: facilitation agents which provide the necessary API interfaces for interaction between old (Web 2.0) and new (Web 3.0) economic models. diff --git a/docs/core-components.md b/docs/core-components.md index 7789ad682a..1ab263f1a5 100644 --- a/docs/core-components.md +++ b/docs/core-components.md @@ -54,7 +54,7 @@ The skills could then read the internal state of the agent, including the agent' A skill encapsulates implementations of the abstract base classes `Handler`, `Behaviour`, and `Task`: -* `Handler`: each skill has none, one or more `Handler` objects, each responsible for the registered messaging protocol. Handlers implement agents' reactive behaviour. If the agent understands the protocol referenced in a received `Envelope`, the `Handler` reacts appropriately to the corresponding message. Each `Handler` is responsible for only one protocol. +* `Handler`: each skill has none, one or more `Handler` objects, each responsible for the registered messaging protocol. Handlers implement agents' reactive behaviour. If the agent understands the protocol referenced in a received `Envelope`, the `Handler` reacts appropriately to the corresponding message. Each `Handler` is responsible for only one protocol. A `Handler` is also capable of dealing with internal messages. * `Behaviour`: none, one or more `Behaviours` encapsulate actions that cause interactions with other agents initiated by the agent. Behaviours implement agents' proactiveness. * `Task`: none, one or more Tasks encapsulate background work internal to the agent. @@ -83,6 +83,8 @@ It is responsible for the agent's crypto-economic security and goal management, By default for every skill, each `Handler`, `Behaviour` and `Task` is registered in the `Filter`. However, note that skills can de-register and re-register themselves. +The `Filter` also routes internal messages from the `DecisionMaker` to the respective `Handler` in the skills. + ## Resource The `Resource` component is made up of `Registries` for each type of resource (e.g. `Protocol`, `Handler`, `Behaviour`, `Task`). diff --git a/docs/index.md b/docs/index.md index af897fe5eb..9b324b8e84 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,37 +1,27 @@ -Do you want to create omnipresent, self-learning, and autonomous agents whose sole existence is to enrich your life? +## Software to work for you -Universal and ubiquitous AEA framework agents work constantly for your benefit without you having to do anything more than write them and start them up. +Do you want to create software to work for you and enrich your life? -They are self learning and self managing with the single goal of ensuring economic gain for their owners. +Autonomous Economic Agents - or AEAs - work continously for your benefit without you having to do anything more than write them and start them up. -Not just agency, AEA framework agents can represent a wide range of application areas. - -Bridging Web 2.0 to Web 3.0, the AEA is the future, now. +AEAs are able to act independent of your constant input and to autonomously develop new capabilities. Their goal is to create economic gain for you, their owner. +AEAs have a wide range of application areas. Check out the demo section for examples. +Bridging Web 2.0 to Web 3.0, AEAs are the future, now. ## More specifically -The AEA framework provides the tools for creating autonomous economic agents (AEA). +The AEA framework provides the tools for creating autonomous economic agents. -It is a Python-based development suite which equips developers with an efficient and easy to understand set of tools for building autonomous economic agents. +It is a Python-based development suite which equips you with an efficient and easy to understand set of tools for building autonomous economic agents. The framework is super modular, easily extensible, and highly composable. The AEA framework attempts to make agent development as straightforward as web development using popular web frameworks. -The AEA super power is their ability to autonomously acquire new skills. - -AEAs achieve their goals with the help of the Fetch.ai OEF and the Fetch.ai Ledger. Third party systems, such as Ethereum, may also allow AEA integration, the bridge to Web 3.0. - - - - - - - - +AEAs achieve their goals with the help of the Fetch.ai OEF - a search and discovery platform for agents - and the Fetch.ai blockchain. Third party systems, such as Ethereum, may also allow AEA integration. !!! Note diff --git a/docs/oef-ledger.md b/docs/oef-ledger.md index 7bffa571c8..b0266a6669 100644 --- a/docs/oef-ledger.md +++ b/docs/oef-ledger.md @@ -1,4 +1,4 @@ -In the AEA framework universe, agents run alongside OEF search nodes against the Fetch.ai ledger and external ledger systems. +In the AEA framework universe, agents run alongside OEF search and discovery nodes against the Fetch.ai ledger and external ledger systems.
![The AEA, OEF, and Ledger systems](assets/oef-ledger.png)
\ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md index 1f47a7a1ca..c4783ad867 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -74,8 +74,7 @@ sudo apt-get install python3.7-dev ## Echo skill demo -The echo skill is a simple demo that prints logs from the agent's main loop as it calls registered `Task` and `Behaviour` code. - +The echo skill is a simple demo that introduces you to the main business logic components of an AEA. ### Download the scripts and packages directories. @@ -85,61 +84,36 @@ svn export https://github.com/fetchai/agents-aea.git/trunk/packages ``` ### Create a new agent + +First create a new agent project and enter it. ``` bash aea create my_first_agent +cd my_first_agent ``` ### Add the echo skill ``` bash -cd my_first_agent + aea add skill echo ``` This copies the echo application code for the behaviours, handlers, and tasks into the skill, ready to run. - -### Add a local connection - -``` bash -aea add connection local -``` - -A local connection provides a local stub for an OEF node instance. - -### Run the agent - -Run the agent with the `local` connection. - -``` bash -aea run --connection local -``` - -You will see the echo task running in the terminal window. - -
![The echo call and response log](assets/echo.png)
- -The framework first calls the `setup` method on the `Handler`, `Behaviour`, and `Task` code in that order; after which it repeatedly calls the `Behaviour` and `Task` methods, `act` and `execute`. This is the main agent loop in action. - -Let's look at the `Handler` in more depth. First, stop the agent. - -### Stop the agent - -Stop the agent by pressing `CTRL c` - ### Add a stub connection +AEAs use messages for communication. We will add a stub connection to send messages to and receive messages from the AEA. + ``` bash aea add connection stub ``` + A stub connection provides an I/O reader/writer. -It uses two files for communication: one for the incoming messages and -the other for the outgoing messages. Each line contains an encoded envelope. +It uses two files for communication: one for the incoming messages and the other for the outgoing messages. Each line contains an encoded envelope. -The agent waits for new messages posted to the file `my_first_agent/input_file`, -and adds a response to the file `my_first_agent/output_file`. +The AEA waits for new messages posted to the file `my_first_agent/input_file`, and adds a response to the file `my_first_agent/output_file`. The format of each line is the following: @@ -153,18 +127,28 @@ For example: recipient_agent,sender_agent,default,{"type": "bytes", "content": "aGVsbG8="} ``` -### Add the line to the input file +### Run the agent + +Run the agent with the `stub` connection. ``` bash -echo 'my_first_agent,sender_agent,default,{"type": "bytes", "content": "aGVsbG8="}' >> input_file +aea run --connection stub ``` -### Run the agent +You will see the echo task running in the terminal window. -Run the agent with the `stub` connection. +
![The echo call and response log](assets/echo.png)
+ +The framework first calls the `setup` method on the `Handler`, `Behaviour`, and `Task` code in that order; after which it repeatedly calls the `Behaviour` and `Task` methods, `act` and `execute`. This is the main agent loop in action. + +Let's look at the `Handler` in more depth. + +### Add a message to the input file + +We send the AEA a message wrapped in an envelope via the input file. ``` bash -aea run --connection stub +echo 'my_first_agent,sender_agent,default,{"type": "bytes", "content": "aGVsbG8="}' >> input_file ``` You will see the `Echo Handler` dealing with the envelope and responding with the same message to the `output_file`, and also decoding the Base64 encrypted message in this case. @@ -184,7 +168,7 @@ info: Echo Task: execute method called. ### Stop the agent -Stop the agent by pressing `CTRL c` +Stop the agent by pressing `CTRL C` ### Delete the agent diff --git a/docs/trust.md b/docs/trust.md index de41eb22b0..0e94045207 100644 --- a/docs/trust.md +++ b/docs/trust.md @@ -1,11 +1,9 @@ AEA applications operate within different orders of trustlessness. -For example, using the AEA weather skill application without a ledger for transactions means that clients must 100% trust the weather station that any data it sends is sufficient, including no data at all. +For example, using the AEA weather skills demo without a ledger means that clients must trust the weather station that any data it sends is sufficient, including no data at all. Similarly, the weather station must trust the weather clients to send payment via some mechanism. -A step up, if you run the weather skill application on a ledger system then the client must again trust the weather station to send sufficient data. However, all transactions are recorded so there is some data verifiability. +A step up, if you run the weather skills demo with a ledger (Fetch.ai or Ethereum) then the clients must again trust the weather station to send sufficient data. However, all payment transactions are executed via the public ledger. And so the weather station must no longer trust the weather clients as it can observe the transaction taking place on the public ledger. -Crucially, the weather station does not need to trust the weather client as it can observe the transaction taking place on the public ledger. +One could expand trustlessness even further by incorporating a third party as an arbitrator or some escrow contract. However, in the weather skills demo there are limits to trustlessness as the station ultimately offers unverifiable data. -An app could expand trustlessness even further by implementing a third party escrow contract. - -Finally, in the case of (non-fungible) token transactions where there is an atomic swap, full trustlessness is apparent. \ No newline at end of file +Finally, in the case of (non-fungible) token transactions where there is an atomic swap, full trustlessness is apparent. This is demonstrated in the TAC. \ No newline at end of file diff --git a/docs/weather-skills.md b/docs/weather-skills.md index 2fc74187be..2f63c6ea7b 100644 --- a/docs/weather-skills.md +++ b/docs/weather-skills.md @@ -1,4 +1,4 @@ -The AEA weather skill demonstrates the interaction between two AEA agents; one as the provider of weather data, the other as the seller of weather data. +The AEA weather skills demonstrate an interaction between two AEAs; one as the provider of weather data (the weather station), the other as the seller of weather data (the weather client). ## Prerequisites @@ -15,7 +15,7 @@ If not, update with the following. pip install aea[all] --force --no-cache-dir ``` -## Demo instructions +## Demo preliminaries Follow the Preliminaries and Installation instructions here. @@ -26,15 +26,20 @@ svn export https://github.com/fetchai/agents-aea.git/trunk/packages svn export https://github.com/fetchai/agents-aea.git/trunk/scripts ``` - -### Launch the OEF Node: +## Launch the OEF Node (for search and discovery): In a separate terminal, launch an OEF node locally: ``` bash python scripts/oef/launch.py -c ./scripts/oef/launch_config.json ``` -### Create the weather station agent: -In the root directory, create the weather station agent. +Keep it running for all the following demos. + +## Demo 1: no ledger payment + +The AEAs negotiate and then transfer the data. No payment takes place. This demo serves as a demonstration of the negotiation steps. + +### Create the weather station AEA: +In the root directory, create the weather station AEA. ``` bash aea create my_weather_station ``` @@ -47,15 +52,14 @@ aea add skill weather_station ``` -### Run the weather station agent - +### Run the weather station AEA ``` bash aea run ``` -### Create the weather client agent -In a new terminal window, return to the root directory and create the weather client agent. +### Create the weather client AEA +In a new terminal window, return to the root directory and create the weather client AEA. ``` bash aea create my_weather_client ``` @@ -68,22 +72,22 @@ aea add skill weather_client ``` -### Run the weather client agent +### Run the weather client AEA ``` bash aea run ``` -### Observe the logs of both agents +### Observe the logs of both AEAs
![Weather station logs](assets/weather-station-logs.png)
![Weather client logs](assets/weather-client-logs.png)
-### Delete the agents +### Delete the AEAs -When you're done, go up a level and delete the agents. +When you're done, go up a level and delete the AEAs. ``` bash cd .. @@ -92,37 +96,36 @@ aea delete my_weather_client ``` -## Using the ledger +## Demo 2: Fetch.ai ledger payment -To run the same example but with a true ledger transaction, do the following. - -### Launch the OEF Node - -``` bash -python scripts/oef/launch.py -c ./scripts/oef/launch_config.json -``` +A demo to run the same scenario but with a true ledger transaction on Fetch.ai test net. This demo assumes the weather client trusts the weather station to send the weather data upon successful payment. -### Create a weather station (ledger version) +### Create the weather station (ledger version) -In a new terminal window, create the agent that will provide weather measurements. +Create the AEA that will provide weather measurements. ``` bash -aea create weather_station -cd weather_station +aea create my_weather_station +cd my_weather_station aea add skill weather_station_ledger ``` ### Create the weather client (ledger version): -In another terminal, create the agent that will query the weather station +In another terminal, create the AEA that will query the weather station ``` bash -aea create weather_client -cd weather_client +aea create my_weather_client +cd my_weather_client aea add skill weather_client_ledger ``` -### Update the agent configs +Additionally, create the private key for the weather client AEA +```bash +aea generate-key fetchai +``` + +### Update the AEA configs Both in `weather_station/aea-config.yaml` and `weather_client/aea-config.yaml`, replace `ledger_apis: []` with: @@ -135,23 +138,123 @@ ledger_apis: port: 80 ``` -### Run the agents +### Fund the weather client AEA + +Create some wealth for your weather client on the Fetch.ai test net (it takes a while): +``` bash +cd .. +python scripts/fetchai_wealth_generation.py --private-key weather_client/fet_private_key.txt --amount 10000000 --addr alpha.fetch-ai.com --port 80 +cd my_weather_client +``` + +### Run the AEAs + +Run both AEAs, from their respective terminals ``` bash aea run ``` -### Generate the private key +You will see that the AEAs negotiate and then transact using the Fetch.ai test net. + +### Delete the AEAs + +When you're done, go up a level and delete the AEAs. + ``` bash -aea generate-key fetchai +cd .. +aea delete my_weather_station +aea delete my_weather_client ``` -### Fund the client agent +## Demo 3: Ethereum ledger payment + +A demo to run the same scenario but with a true ledger transaction on Fetch.ai test net. This demo assumes the weather client trusts the weather station to send the weather data upon successful payment. + +### Create the weather station (ledger version) + +Create the AEA that will provide weather measurements. -After you run the client and generate the private key, send your weather client some FET with its FET address (it takes a while): ``` bash -python scripts/fetchai_wealth_generation.py --private-key weather_client/fet_private_key.txt --amount 10000000 --addr alpha.fetch-ai.com --port 80 +aea create my_weather_station +cd my_weather_station +aea add skill weather_station_ledger +``` + +### Create the weather client (ledger version): + +In another terminal, create the AEA that will query the weather station + +``` bash +aea create my_weather_client +cd my_weather_client +aea add skill weather_client_ledger +``` + +Additionally, create the private key for the weather client AEA +```bash +aea generate-key ethereum +``` + +### Update the AEA configs + +Both in `weather_station/aea-config.yaml` and +`weather_client/aea-config.yaml`, replace `ledger_apis: []` with: + +``` yaml +ledger_apis: + - ledger_api: + addr: https://ropsten.infura.io/v3/f00f7b3ba0e848ddbdc8941c527447fe + ledger: ethereum + port: 3 +``` + +### Update the skill configs + +In the weather station skill config (`my_weather_station/skills/weather_station_ledger/skill.yaml`) under strategy change the `currency_pbk` and `ledger_id` as follows: +``` +currency_pbk: 'ETH' +ledger_id: 'ethereum' +``` +and under ledgers change to: +``` +ledgers: ['ethereum'] +``` + +In the weather client skill config (`my_weather_client/skills/weather_client_ledger/skill.yaml`) under strategy change the `currency_pbk` and `ledger_id` as follows: +``` +max_buyer_tx_fee: 20000 +currency_pbk: 'ETH' +ledger_id: 'ethereum' +``` +and under ledgers change to: +``` +ledgers: ['ethereum'] +``` + +### Fund the weather client AEA + +Create some wealth for your weather client on the Ethereum Ropsten test net: + +Go to Metamask [Faucet](https://faucet.metamask.io) and request some test ETH for the account your weather client AEA is using (you need to first load your AEAs private key into MetaMask). Your private key is at `weather_client/eth_private_key.txt`. + +### Run the AEAs + +Run both AEAs, from their respective terminals +``` bash +aea run ``` +You will see that the AEAs negotiate and then transact using the Fetch.ai test net. + +### Delete the AEAs + +When you're done, go up a level and delete the AEAs. + +``` bash +cd .. +aea delete my_weather_station +aea delete my_weather_client +```
diff --git a/examples/echo_skill/README.md b/examples/echo_skill/README.md deleted file mode 100644 index e4fb84e13c..0000000000 --- a/examples/echo_skill/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# echo_skill - -A guide to create an AEA with the echo_skill. - -## Quick start - -This quick start explains how to create and launch an agent with the cli. - -- in any directory, open a terminal and execute: - - aea create my_first_agent - - This command will create a directory named `my_first_agent`. It will further create the `my_first_agent/skills` folder, with the `error` skill package inside. It will also create the `my_first_agent/protocols` folder, with the `default` protocol package inside. Finally, it will create the `my_first_agent/connections` folder, with the `oef` connection package inside. - -- enter into the agent's directory: - - cd my_first_agent - -- add a skill to the agent, e.g.: - - aea add skill echo - - This command will add the `echo` skill package to the `my_first_agent/skills` folder. - -- start an oef from a separate terminal: - - python scripts/oef/launch.py -c ./scripts/oef/launch_config.json - -- Run the agent. Assuming an OEF node is running at `127.0.0.1:10000` - - aea run - -- For debugging run with: - - aea -v DEBUG run - -- Press CTRL+C to stop the execution. - -- Delete the agent: - - cd .. - aea delete my_first_agent - diff --git a/examples/gym_ex/proxy/agent.py b/examples/gym_ex/proxy/agent.py index 1dbde96023..b35f41f12a 100644 --- a/examples/gym_ex/proxy/agent.py +++ b/examples/gym_ex/proxy/agent.py @@ -47,7 +47,7 @@ def __init__(self, name: str, gym_env: gym.Env, proxy_env_queue: Queue) -> None: :param proxy_env_queue: the queue of the proxy environment :return: None """ - wallet = Wallet({DEFAULT: None}, {}) + wallet = Wallet({DEFAULT: None}) super().__init__(name, wallet, timeout=0) self.proxy_env_queue = proxy_env_queue crypto_object = self.wallet.crypto_objects.get(DEFAULT) diff --git a/examples/gym_skill/README.md b/examples/gym_skill/README.md deleted file mode 100644 index 7f28bd12f7..0000000000 --- a/examples/gym_skill/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# gym_skill - -A guide to create an AEA with the gym_skill. - -## Quick start - -- Create an agent: - - aea create my_gym_agent - -- Cd into agent: - - cd my_gym_agent - -- Add the 'gym' skill: - - aea add skill gym - -- Copy the gym environment to the agent directory: - - mkdir gyms - cp -a ../examples/gym_ex/gyms/. gyms/ - -- Add a gym connection: - - aea add connection gym - -- Update the connection config `my_gym_agent/connections/gym/connection.yaml`: - - env: gyms.env.BanditNArmedRandom - -- Run the agent with the 'gym' connection: - - aea run --connection gym - -- Delete the agent: - - cd .. - aea delete my_gym_agent diff --git a/examples/weather_skills/README.md b/examples/weather_skills/README.md deleted file mode 100644 index 51e624f1c4..0000000000 --- a/examples/weather_skills/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Weather station and client example - -A guide to create two AEAs, one a weather station selling weather data, another a -purchaser (client) of weather data. This setup assumes the weather client trusts the weather station -to send the data upon successful payment. - -## Without ledger - -The AEAs negotiate and then transfer the data. No payment takes place. - -- Launch the OEF Node (for search and discovery): - - python scripts/oef/launch.py -c ./scripts/oef/launch_config.json - -- Create a weather station - the agent that will provide weather measurements: - - aea create weather_station - cd weather_station - aea add skill weather_station - aea run - -- In another terminal, create the weather client - the agent that will query the weather station - - aea create weather_client - cd weather_client - aea add skill weather_client - aea run - -- Afterwards, clean up: - - cd .. - aea delete weather_station - aea delete weather_client - - -## With ledger (Fetch.ai) - -The AEAs use the Fetch.ai ledger to settle their trade. - -- Launch the OEF Node (for search and discovery): - - python scripts/oef/launch.py -c ./scripts/oef/launch_config.json - -- Create a weather station (ledger version) - the agent that will provide weather measurements: - - aea create weather_station - cd weather_station - aea add skill weather_station_ledger - -- In another terminal, create the weather client (ledger version) - the agent that will query the weather station - - aea create weather_client - cd weather_client - aea add skill weather_client_ledger - -- Generate the private key for the weather client: - - aea generate-key fetchai - -- Both in `weather_station/aea-config.yaml` and -`weather_client/aea-config.yaml`, replace `ledger_apis: []` with: -``` -ledger_apis: -- ledger_api: - addr: alpha.fetch-ai.com - ledger: fetchai - port: 80 -``` - -- Generate some wealth to your weather client FET address (it takes a while): -``` -cd .. -python scripts/fetchai_wealth_generation.py --private-key weather_client/fet_private_key.txt --amount 10000000 --addr alpha.fetch-ai.com --port 80 -cd weather_client -``` - -- Run both agents, from their respective terminals: - - aea run - -- Afterwards, clean up: - - cd .. - aea delete weather_station - aea delete weather_client - -## With ledger (Ethereum) - -The AEAs use the Ethereum ledger to settle their trade. - -- Follow the first three steps from the previous section. - -- Generate the private key for the weather client: - - aea generate-key ethereum - -- Both in `weather_station/aea-config.yaml` and -`weather_client/aea-config.yaml`, replace `ledger_apis: []` with: -``` -- ledger_api: - addr: https://ropsten.infura.io/v3/f00f7b3ba0e848ddbdc8941c527447fe - ledger: ethereum - port: 3 -``` - -- In weather station client skill config under strategy change to -``` -currency_pbk: 'ETH' -ledger_id: 'ethereum' -``` -and under ledgers change to: -``` -ledgers: ['ethereum'] -``` - -- Generate some wealth to your weather client ETH address: - -Go to Metamask [Faucet](https://faucet.metamask.io) and request some test ETH for the account your AEA is using (you need to first load your AEAs private key into MetaMask). Your private key is at `weather_client/eth_private_key.txt`. - -- Follow the last two stesp from the previous section. diff --git a/mkdocs.yml b/mkdocs.yml index 9cf3367cb1..3b4fc9afc2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,15 +25,15 @@ nav: - "Application areas": 'app-areas.md' - "Relation to OEF and Ledger": 'oef-ledger.md' - "Trust issues": 'trust.md' - - "Two-layered machine learning": 'two-layer.md' + # - "Two-layered machine learning": 'two-layer.md' - Demos: - "Gym demo": 'gym-plugin.md' - "Gym skill": 'gym-skill.md' - "Weather skills": 'weather-skills.md' - "TAC external app": 'tac.md' - - "FIPA skill": 'fipa-skill.md' - - "TAC skill": 'tac-skill.md' - # - "Car park agent": 'car-park.md' + # - "FIPA skill": 'fipa-skill.md' + # - "TAC skill": 'tac-skill.md' + # - "Car park agent": 'car-park.md' - Architecture: - "Design principles": 'design-principles.md' - "Architectural diagram": 'diagram.md' From cf9edaf04c59a21f6f2ded5cf30c9a31c5cab30d Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 12:27:07 +0000 Subject: [PATCH 15/38] Update the weather_station_example to work with dialogues and strategies --- packages/skills/weather_client/behaviours.py | 57 +++- packages/skills/weather_client/dialogues.py | 214 ++++++++++++++ packages/skills/weather_client/handlers.py | 266 ++++++++++++------ packages/skills/weather_client/skill.yaml | 35 +-- packages/skills/weather_client/strategy.py | 111 ++++++++ packages/skills/weather_client/tasks.py | 50 ---- packages/skills/weather_station/behaviours.py | 85 ++++-- packages/skills/weather_station/dialogues.py | 228 +++++++++++++++ .../dummy_weather_station_data.py | 4 +- packages/skills/weather_station/handlers.py | 216 +++++++++----- packages/skills/weather_station/skill.yaml | 28 +- packages/skills/weather_station/strategy.py | 138 +++++++++ .../weather_station_data_model.py | 1 - 13 files changed, 1161 insertions(+), 272 deletions(-) create mode 100644 packages/skills/weather_client/dialogues.py create mode 100644 packages/skills/weather_client/strategy.py delete mode 100644 packages/skills/weather_client/tasks.py create mode 100644 packages/skills/weather_station/dialogues.py create mode 100644 packages/skills/weather_station/strategy.py diff --git a/packages/skills/weather_client/behaviours.py b/packages/skills/weather_client/behaviours.py index 26d6dd5f6f..37f876ec00 100644 --- a/packages/skills/weather_client/behaviours.py +++ b/packages/skills/weather_client/behaviours.py @@ -18,16 +18,24 @@ # ------------------------------------------------------------------------------ """This package contains a scaffold of a behaviour.""" +import logging +from typing import cast, TYPE_CHECKING +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI from aea.protocols.oef.message import OEFMessage -from aea.protocols.oef.models import Query, Constraint, ConstraintType from aea.protocols.oef.serialization import DEFAULT_OEF, OEFSerializer from aea.skills.base import Behaviour -REQUEST_ID = 1 +if TYPE_CHECKING: + from packages.skills.weather_client.strategy import Strategy +else: + from weather_client_skill.strategy import Strategy +logger = logging.getLogger("aea.weather_client_ledger_skill") -class MyBuyBehaviour(Behaviour): + +class MySearchBehaviour(Behaviour): """This class scaffolds a behaviour.""" def __init__(self, **kwargs): @@ -36,16 +44,21 @@ def __init__(self, **kwargs): def setup(self) -> None: """Implement the setup for the behaviour.""" - search_query_empty_model = Query([Constraint("country", - ConstraintType("==", "UK"))], model=None) - search_request = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, - id=REQUEST_ID, - query=search_query_empty_model) + if self.context.ledger_apis.has_fetchai: + fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + if fet_balance > 0: + logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) + else: + logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) + # TODO: deregister skill from filter - self.context.outbox.put_message(to=DEFAULT_OEF, - sender=self.context.agent_public_key, - protocol_id=OEFMessage.protocol_id, - message=OEFSerializer().encode(search_request)) + if self.context.ledger_apis.has_ethereum: + eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + if eth_balance > 0: + logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) + else: + logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) + # TODO: deregister skill from filter def act(self) -> None: """ @@ -53,7 +66,17 @@ def act(self) -> None: :return: None """ - pass + strategy = cast(Strategy, self.context.strategy) + if strategy.is_time_to_search(): + query = strategy.get_service_query() + search_id = strategy.get_next_search_id() + oef_msg = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, + id=search_id, + query=query) + self.context.outbox.put_message(to=DEFAULT_OEF, + sender=self.context.agent_public_key, + protocol_id=OEFMessage.protocol_id, + message=OEFSerializer().encode(oef_msg)) def teardown(self) -> None: """ @@ -61,4 +84,10 @@ def teardown(self) -> None: :return: None """ - pass + if self.context.ledger_apis.has_fetchai: + balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + + if self.context.ledger_apis.has_ethereum: + balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) diff --git a/packages/skills/weather_client/dialogues.py b/packages/skills/weather_client/dialogues.py new file mode 100644 index 0000000000..bb6b721095 --- /dev/null +++ b/packages/skills/weather_client/dialogues.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +""" +This module contains the classes required for dialogue management. + +- DialogueLabel: The dialogue label class acts as an identifier for dialogues. +- Dialogue: The dialogue class maintains state of a dialogue and manages it. +- Dialogues: The dialogues class keeps track of all dialogues. +""" + +from enum import Enum +import logging +from typing import Dict, Optional, cast + +from aea.helpers.dialogue.base import DialogueLabel +from aea.helpers.dialogue.base import Dialogue as BaseDialogue +from aea.mail.base import Address +from aea.protocols.base import Message +from aea.protocols.fipa.message import FIPAMessage, VALID_PREVIOUS_PERFORMATIVES +from aea.protocols.oef.models import Description +from aea.skills.base import SharedClass + +logger = logging.getLogger("aea.weather_client_ledger_skill") + + +class Dialogue(BaseDialogue): + """The dialogue class maintains state of a dialogue and manages it.""" + + class EndState(Enum): + """This class defines the end states of a dialogue.""" + + SUCCESSFUL = 0 + DECLINED_CFP = 1 + DECLINED_ACCEPT = 2 + + def __init__(self, dialogue_label: DialogueLabel, **kwargs) -> None: + """ + Initialize a dialogue label. + + :param dialogue_label: the identifier of the dialogue + + :return: None + """ + BaseDialogue.__init__(self, dialogue_label=dialogue_label) + self.proposal = None # type: Optional[Description] + + def is_valid_next_message(self, fipa_msg: FIPAMessage) -> bool: + """ + Check whether this is a valid next message in the dialogue. + + :return: True if yes, False otherwise. + """ + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = cast(FIPAMessage.Performative, fipa_msg.get("performative")) + last_outgoing_message = self.last_outgoing_message + if last_outgoing_message is None: + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP + else: + last_message_id = cast(int, last_outgoing_message.get("message_id")) + last_target = cast(int, last_outgoing_message.get("target")) + last_performative = cast(FIPAMessage.Performative, last_outgoing_message.get("performative")) + result = this_message_id == last_message_id + 1 and \ + this_target == last_target + 1 and \ + last_performative in VALID_PREVIOUS_PERFORMATIVES[this_performative] + return result + + +class DialogueStats(object): + """Class to handle statistics on the negotiation.""" + + def __init__(self) -> None: + """Initialize a StatsManager.""" + self._self_initiated = {Dialogue.EndState.SUCCESSFUL: 0, + Dialogue.EndState.DECLINED_CFP: 0, + Dialogue.EndState.DECLINED_ACCEPT: 0} # type: Dict[Dialogue.EndState, int] + + @property + def self_initiated(self) -> Dict[Dialogue.EndState, int]: + """Get the stats dictionary on self initiated dialogues.""" + return self._self_initiated + + def add_dialogue_endstate(self, end_state: Dialogue.EndState) -> None: + """ + Add dialogue endstate stats. + + :param end_state: the end state of the dialogue + :return: None + """ + self._self_initiated[end_state] += 1 + + +class Dialogues(SharedClass): + """The dialogues class keeps track of all dialogues.""" + + def __init__(self, **kwargs) -> None: + """ + Initialize dialogues. + + :return: None + """ + SharedClass.__init__(self, **kwargs) + self._dialogues = {} # type: Dict[DialogueLabel, Dialogue] + self._dialogue_id = 0 + self._dialogue_stats = DialogueStats() + + @property + def dialogues(self) -> Dict[DialogueLabel, Dialogue]: + """Get dictionary of dialogues in which the agent is engaged in.""" + return self._dialogues + + @property + def dialogue_stats(self) -> DialogueStats: + """Get the dialogue statistics.""" + return self._dialogue_stats + + def _next_dialogue_id(self) -> int: + """ + Increment the id and returns it. + + :return: the next id + """ + self._dialogue_id += 1 + return self._dialogue_id + + def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> bool: + """ + Check whether an agent message is part of a registered dialogue. + + :param fipa_msg: the fipa message + :param sender: the sender + :param agent_pbk: the public key of the agent + + :return: boolean indicating whether the message belongs to a registered dialogue + """ + fipa_msg = cast(FIPAMessage, fipa_msg) + dialogue_id = cast(int, fipa_msg.get("dialogue_id")) + self_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, agent_pbk) + if self_initiated_dialogue_label in self.dialogues: + self_initiated_dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) + result = self_initiated_dialogue.is_valid_next_message(fipa_msg) + else: + result = False + return result + + def get_dialogue(self, dialogue_id: int, sender: Address, agent_pbk: Address) -> Dialogue: + """ + Retrieve dialogue. + + :param dialogue_id: the dialogue id + :param sender_pbk: the sender public key + :param agent_pbk: the public key of the agent + + :return: the dialogue + """ + self_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, agent_pbk) + dialogue = cast(Dialogue, self.dialogues[self_initiated_dialogue_label]) + return dialogue + + def create_self_initiated(self, dialogue_opponent_pbk: Address, dialogue_starter_pbk: Address) -> Dialogue: + """ + Create a self initiated dialogue. + + :param dialogue_opponent_pbk: the pbk of the agent with which the dialogue is kept. + :param dialogue_starter_pbk: the pbk of the agent which started the dialogue + + :return: the created dialogue. + """ + dialogue_label = DialogueLabel(self._next_dialogue_id(), dialogue_opponent_pbk, dialogue_starter_pbk) + result = self._create(dialogue_label) + return result + + def _create(self, dialogue_label: DialogueLabel) -> Dialogue: + """ + Create a dialogue. + + :param dialogue_label: the dialogue label + :param is_seller: boolean indicating the agent role + + :return: the created dialogue + """ + assert dialogue_label not in self.dialogues + dialogue = Dialogue(dialogue_label) + self.dialogues.update({dialogue_label: dialogue}) + return dialogue + + def reset(self) -> None: + """ + Reset the dialogues. + + :return: None + """ + self._dialogues = {} + self._dialogue_stats = DialogueStats() diff --git a/packages/skills/weather_client/handlers.py b/packages/skills/weather_client/handlers.py index 21d875b596..5004f61bba 100644 --- a/packages/skills/weather_client/handlers.py +++ b/packages/skills/weather_client/handlers.py @@ -18,24 +18,28 @@ # ------------------------------------------------------------------------------ """This package contains a scaffold of a handler.""" - import logging -from typing import Optional, cast, List +import pprint +from typing import List, Optional, cast, TYPE_CHECKING from aea.configurations.base import ProtocolId from aea.protocols.base import Message from aea.protocols.default.message import DefaultMessage +from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.models import Description from aea.skills.base import Handler -MAX_PRICE = 2 -STARTING_MESSAGE_ID = 1 -STARTING_TARGET_ID = 0 +if TYPE_CHECKING: + from packages.skills.weather_client.dialogues import Dialogue, Dialogues + from packages.skills.weather_client.strategy import Strategy +else: + from weather_client_skill.dialogues import Dialogue, Dialogues + from weather_client_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_client_skill") +logger = logging.getLogger("aea.weather_client_ledger_skill") class FIPAHandler(Handler): @@ -43,11 +47,6 @@ class FIPAHandler(Handler): SUPPORTED_PROTOCOL = FIPAMessage.protocol_id # type: Optional[ProtocolId] - def __init__(self, **kwargs): - """Initiliase the handler.""" - super().__init__(**kwargs) - self.max_price = kwargs['max_price'] if 'max_price' in kwargs.keys() else MAX_PRICE - def setup(self) -> None: """ Implement the setup. @@ -64,19 +63,32 @@ def handle(self, message: Message, sender: str) -> None: :param sender: the sender :return: None """ + # convenience representations + fipa_msg = cast(FIPAMessage, message) msg_performative = FIPAMessage.Performative(message.get('performative')) - proposals = cast(List[Description], message.get("proposal")) message_id = cast(int, message.get("message_id")) dialogue_id = cast(int, message.get("dialogue_id")) + + # recover dialogue + dialogues = cast(Dialogues, self.context.dialogues) + logger.info(fipa_msg) + if dialogues.is_belonging_to_registered_dialogue(fipa_msg, sender, self.context.agent_public_key): + dialogue = dialogues.get_dialogue(dialogue_id, sender, self.context.agent_public_key) + dialogue.incoming_extend(fipa_msg) + else: + self._handle_unidentified_dialogue(fipa_msg, sender) + return + + # handle message if msg_performative == FIPAMessage.Performative.PROPOSE: - if proposals is not []: - for item in proposals: - logger.info("[{}]: received proposal={} in dialogue={}".format(self.context.agent_name, item.values, dialogue_id)) - if "Price" in item.values.keys(): - if item.values["Price"] < self.max_price: - self.handle_accept(sender, message_id, dialogue_id) - else: - self.handle_decline(sender, message_id, dialogue_id) + self._handle_propose(fipa_msg, sender, message_id, dialogue_id, dialogue) + elif msg_performative == FIPAMessage.Performative.DECLINE: + self._handle_decline(fipa_msg, sender, message_id, dialogue_id, dialogue) + elif msg_performative == FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS: + self._handle_match_accept(fipa_msg, sender, message_id, dialogue_id, dialogue) + elif msg_performative == FIPAMessage.Performative.INFORM: + logger.info("RECEIVED AN INFORM MESSAGE!!!!") + self._handle_inform(fipa_msg, sender, message_id, dialogue_id, dialogue) def teardown(self) -> None: """ @@ -86,100 +98,142 @@ def teardown(self) -> None: """ pass - def handle_accept(self, sender: str, message_id: int, dialogue_id: int): + def _handle_unidentified_dialogue(self, msg: FIPAMessage, sender: str) -> None: """ - Handle sending accept message. + Handle an unidentified dialogue. - :param sender: the sender of the message - :param message_id: the message id - :param dialogue_id: the dialogue id + :param msg: the message + :param sender: the sender """ - new_message_id = message_id + 1 - new_target_id = message_id - logger.info("[{}]: accepting the proposal from sender={}".format(self.context.agent_name, sender)) - msg = FIPAMessage(message_id=new_message_id, - dialogue_id=dialogue_id, - target=new_target_id, - performative=FIPAMessage.Performative.ACCEPT) + logger.info("[{}]: unidentified dialogue.".format(self.context.agent_name)) + default_msg = DefaultMessage(type=DefaultMessage.Type.ERROR, + error_code=DefaultMessage.ErrorCode.INVALID_DIALOGUE.value, + error_msg="Invalid dialogue.", + error_data="fipa_message") # TODO: send back FIPASerializer().encode(msg)) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, - protocol_id=FIPAMessage.protocol_id, - message=FIPASerializer().encode(msg)) + protocol_id=DefaultMessage.protocol_id, + message=DefaultSerializer().encode(default_msg)) - def handle_decline(self, sender: str, message_id: int, dialogue_id: int): + def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ - Handle sending decline message. + Handle the propose. - :param sender: the sender of the message + :param msg: the message + :param sender: the sender :param message_id: the message id :param dialogue_id: the dialogue id + :param dialogue: the dialogue object + :return: None """ new_message_id = message_id + 1 new_target_id = message_id - logger.info("[{}]: declinig the proposal from sender={}".format(self.context.agent_name, sender)) - msg = FIPAMessage(message_id=new_message_id, - dialogue_id=dialogue_id, - target=new_target_id, - performative=FIPAMessage.Performative.DECLINE) - self.context.outbox.put_message(to=sender, - sender=self.context.agent_public_key, - protocol_id=FIPAMessage.protocol_id, - message=FIPASerializer().encode(msg)) - - -class OEFHandler(Handler): - """This class scaffolds a handler.""" - - SUPPORTED_PROTOCOL = OEFMessage.protocol_id # type: Optional[ProtocolId] + proposals = cast(List[Description], msg.get("proposal")) + if proposals is not []: + # only take the first proposal + proposal = proposals[0] + logger.info("[{}]: received proposal={} from sender={}".format(self.context.agent_name, + proposal.values, + sender[-5:])) + strategy = cast(Strategy, self.context.strategy) + acceptable = strategy.is_acceptable_proposal(proposal) + if acceptable: + strategy.is_searching = False + logger.info("[{}]: accepting the proposal from sender={}".format(self.context.agent_name, + sender[-5:])) + dialogue.proposal = proposal + accept_msg = FIPAMessage(message_id=new_message_id, + dialogue_id=dialogue_id, + target=new_target_id, + performative=FIPAMessage.Performative.ACCEPT) + dialogue.outgoing_extend(accept_msg) + self.context.outbox.put_message(to=sender, + sender=self.context.agent_public_key, + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(accept_msg)) + else: + logger.info("[{}]: declining the proposal from sender={}".format(self.context.agent_name, + sender[-5:])) + decline_msg = FIPAMessage(message_id=new_message_id, + dialogue_id=dialogue_id, + target=new_target_id, + performative=FIPAMessage.Performative.DECLINE) + dialogue.outgoing_extend(decline_msg) + self.context.outbox.put_message(to=sender, + sender=self.context.agent_public_key, + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(decline_msg)) - def __init__(self, **kwargs): - """Initialise the oef handler.""" - super().__init__(**kwargs) - self.dialogue_id = 1 + def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: + """ + Handle the decline. - def setup(self) -> None: - """Call to setup the handler.""" - pass + :param msg: the message + :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object + :return: None + """ + logger.info("[{}]: received DECLINE from sender={}".format(self.context.agent_name, sender[-5:])) - def handle(self, message: Message, sender: str) -> None: + def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, + dialogue: Dialogue) -> None: """ - Implement the reaction to a message. + Handle the match accept. - :param message: the message + :param msg: the message :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object :return: None """ - msg_type = OEFMessage.Type(message.get("type")) - - if msg_type is OEFMessage.Type.SEARCH_RESULT: - agents = cast(List[str], message.get("agents")) - logger.info("[{}]: found agents={}".format(self.context.agent_name, agents)) - for agent in agents: - msg = FIPAMessage(message_id=STARTING_MESSAGE_ID, - dialogue_id=self.dialogue_id, - performative=FIPAMessage.Performative.CFP, - target=STARTING_TARGET_ID, - query=None - ) - self.dialogue_id += 1 - self.context.outbox.put_message(to=agent, - sender=self.context.agent_public_key, - protocol_id=FIPAMessage.protocol_id, - message=FIPASerializer().encode(msg)) + fipa_msg = cast(FIPAMessage, dialogue.last_incoming_message) + new_message_id = cast(int, fipa_msg.get("message_id")) + 1 + new_target_id = cast(int, fipa_msg.get("target")) + 1 + dialogue_id = cast(int, fipa_msg.get("dialogue_id")) + counterparty_pbk = dialogue.dialogue_label.dialogue_opponent_pbk + inform_msg = FIPAMessage(message_id=new_message_id, + dialogue_id=dialogue_id, + target=new_target_id, + performative=FIPAMessage.Performative.INFORM, + json_data={"transaction": "Done"}) + dialogue.outgoing_extend(inform_msg) + self.context.outbox.put_message(to=counterparty_pbk, + sender=self.context.agent_public_key, + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(inform_msg)) + logger.info("[{}]: informing counterparty={} of transaction digest.".format(self.context.agent_name, + counterparty_pbk[-5:])) + self._received_tx_message = True - def teardown(self) -> None: + def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ - Implement the handler teardown. + Handle the match inform. + :param msg: the message + :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object :return: None """ - pass + logger.info("[{}]: received INFORM from sender={}".format(self.context.agent_name, sender[-5:])) + json_data = cast(dict, msg.get("json_data")) + if 'weather_data' in json_data.keys(): + weather_data = json_data['weather_data'] + logger.info("[{}]: received the following weather data={}".format(self.context.agent_name, + pprint.pformat(weather_data))) + else: + logger.info("[{}]: received no data from sender={}".format(self.context.agent_name, + sender[-5:])) -class DefaultHandler(Handler): +class OEFHandler(Handler): """This class scaffolds a handler.""" - SUPPORTED_PROTOCOL = DefaultMessage.protocol_id # type: Optional[ProtocolId] + SUPPORTED_PROTOCOL = OEFMessage.protocol_id # type: Optional[ProtocolId] def setup(self) -> None: """Call to setup the handler.""" @@ -193,12 +247,13 @@ def handle(self, message: Message, sender: str) -> None: :param sender: the sender :return: None """ - logger.info("[{}]: receiving data ...".format(self.context.agent_name)) - json_data = message.get("content") - if json_data is not None: - logger.info("[{}]: this is the data I got: {}".format(self.context.agent_name, json_data.decode())) - else: - logger.info("[{}]: there is no data in the message!".format(self.context.agent_name)) + # convenience representations + oef_msg = cast(OEFMessage, message) + oef_msg_type = OEFMessage.Type(oef_msg.get("type")) + + if oef_msg_type is OEFMessage.Type.SEARCH_RESULT: + agents = cast(List[str], oef_msg.get("agents")) + self._handle_search(agents) def teardown(self) -> None: """ @@ -207,3 +262,34 @@ def teardown(self) -> None: :return: None """ pass + + def _handle_search(self, agents: List[str]) -> None: + """ + Handle the search response. + + :param agents: the agents returned by the search + :return: None + """ + if len(agents) > 0: + logger.info("[{}]: found agents={}, stopping search.".format(self.context.agent_name, list(map(lambda x: x[-5:], agents)))) + strategy = cast(Strategy, self.context.strategy) + # stopping search + strategy.is_searching = False + # pick first agent found + opponent_pbk = agents[0] + dialogues = cast(Dialogues, self.context.dialogues) + dialogue = dialogues.create_self_initiated(opponent_pbk, self.context.agent_public_key) + query = strategy.get_service_query() + logger.info("[{}]: sending CFP to agent={}".format(self.context.agent_name, opponent_pbk[-5:])) + cfp_msg = FIPAMessage(message_id=FIPAMessage.STARTING_MESSAGE_ID, + dialogue_id=dialogue.dialogue_label.dialogue_id, + performative=FIPAMessage.Performative.CFP, + target=FIPAMessage.STARTING_TARGET, + query=query) + dialogue.outgoing_extend(cfp_msg) + self.context.outbox.put_message(to=opponent_pbk, + sender=self.context.agent_public_key, + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(cfp_msg)) + else: + logger.info("[{}]: found no agents, continue searching.".format(self.context.agent_name)) diff --git a/packages/skills/weather_client/skill.yaml b/packages/skills/weather_client/skill.yaml index 23e0939389..7aea8fba33 100644 --- a/packages/skills/weather_client/skill.yaml +++ b/packages/skills/weather_client/skill.yaml @@ -3,29 +3,30 @@ authors: Fetch.AI Limited version: 0.1.0 license: Apache 2.0 url: "" -description: "The weather client skill looks for weather stations to buy weather data from." behaviours: - behaviour: - class_name: MyBuyBehaviour - args: - foo: bar + class_name: MySearchBehaviour + args: {} handlers: - handler: class_name: FIPAHandler - args: - foo: bar - - handler: - class_name: DefaultHandler - args: - foo: bar + args: {} - handler: class_name: OEFHandler + args: {} +tasks: [] +shared_classes: + - shared_class: + class_name: Strategy args: - foo: bar -tasks: - - task: - class_name: EmptyTask - args: - foo: bar -shared_classes: [] + country: UK + search_interval: 5 + max_row_price: 4 + max_buyer_tx_fee: 1 + currency_pbk: 'FET' + ledger_id: 'fetchai' + - shared_class: + class_name: Dialogues + args: {} protocols: ['fipa','default','oef'] +ledgers: ['fetchai'] diff --git a/packages/skills/weather_client/strategy.py b/packages/skills/weather_client/strategy.py new file mode 100644 index 0000000000..60fa0f3279 --- /dev/null +++ b/packages/skills/weather_client/strategy.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the strategy class.""" + +import datetime +from typing import cast + +from aea.protocols.oef.models import Description, Query, Constraint, ConstraintType +from aea.skills.base import SharedClass + +DEFAULT_COUNTRY = 'UK' +SEARCH_TERM = 'country' +DEFAULT_SEARCH_INTERVAL = 5.0 +DEFAULT_MAX_ROW_PRICE = 5 +DEFAULT_MAX_TX_FEE = 2 +DEFAULT_CURRENCY_PBK = 'FET' +DEFAULT_LEDGER_ID = 'fetchai' + + +class Strategy(SharedClass): + """This class defines a strategy for the agent.""" + + def __init__(self, **kwargs) -> None: + """ + Initialize the strategy of the agent. + + :return: None + """ + self._country = kwargs.pop('country') if 'country' in kwargs.keys() else DEFAULT_COUNTRY + self._search_interval = cast(float, kwargs.pop('search_interval')) if 'search_interval' in kwargs.keys() else DEFAULT_SEARCH_INTERVAL + self._max_row_price = kwargs.pop('max_row_price') if 'max_row_price' in kwargs.keys() else DEFAULT_MAX_ROW_PRICE + self.max_buyer_tx_fee = kwargs.pop('max_tx_fee') if 'max_tx_fee' in kwargs.keys() else DEFAULT_MAX_TX_FEE + self._currency_pbk = kwargs.pop('currency_pbk') if 'currency_pbk' in kwargs.keys() else DEFAULT_CURRENCY_PBK + self._ledger_id = kwargs.pop('ledger_id') if 'ledger_id' in kwargs.keys() else DEFAULT_LEDGER_ID + super().__init__(**kwargs) + self._search_id = 0 + self.is_searching = True + self._last_search_time = datetime.datetime.now() + + def get_next_search_id(self) -> int: + """ + Get the next search id and set the search time. + + :return: the next search id + """ + self._search_id += 1 + self._last_search_time = datetime.datetime.now() + return self._search_id + + def get_service_query(self) -> Query: + """ + Get the service query of the agent. + + :return: the query + """ + query = Query([Constraint(SEARCH_TERM, ConstraintType("==", self._country))], model=None) + return query + + def is_time_to_search(self) -> bool: + """ + Check whether it is time to search. + + :return: whether it is time to search + """ + if not self.is_searching: + return False + now = datetime.datetime.now() + diff = now - self._last_search_time + result = diff.total_seconds() > self._search_interval + return result + + def is_acceptable_proposal(self, proposal: Description) -> bool: + """ + Check whether it is an acceptable proposal. + + :return: whether it is acceptable + """ + result = (proposal.values['price'] - proposal.values['seller_tx_fee'] > 0) and \ + (proposal.values['price'] <= self._max_row_price * proposal.values['rows']) and \ + (proposal.values['currency_pbk'] == self._currency_pbk) and \ + (proposal.values['ledger_id'] == self._ledger_id) + return result + + def is_affordable_proposal(self, proposal: Description) -> bool: + """ + Check whether it is an affordable proposal. + + :return: whether it is affordable + """ + payable = proposal.values['price'] + self.max_buyer_tx_fee + ledger_id = proposal.values['ledger_id'] + address = cast(str, self.context.agent_addresses.get(ledger_id)) + balance = self.context.ledger_apis.token_balance(ledger_id, address) + return balance >= payable diff --git a/packages/skills/weather_client/tasks.py b/packages/skills/weather_client/tasks.py deleted file mode 100644 index ed4a19c622..0000000000 --- a/packages/skills/weather_client/tasks.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""This package contains a scaffold of a task.""" - -from aea.skills.base import Task - - -class EmptyTask(Task): - """This class scaffolds a task.""" - - def setup(self) -> None: - """ - Implement the setup. - - :return: None - """ - pass - - def execute(self) -> None: - """ - Implement the task execution. - - :return: None - """ - pass - - def teardown(self) -> None: - """ - Implement the task teardown. - - :return: None - """ - pass diff --git a/packages/skills/weather_station/behaviours.py b/packages/skills/weather_station/behaviours.py index 75ca85ce5f..3b0da66eff 100644 --- a/packages/skills/weather_station/behaviours.py +++ b/packages/skills/weather_station/behaviours.py @@ -20,32 +20,31 @@ """This package contains a scaffold of a behaviour.""" import logging +from typing import cast, TYPE_CHECKING +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI from aea.skills.base import Behaviour -from typing import TYPE_CHECKING -from aea.protocols.oef.models import Description from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.serialization import OEFSerializer, DEFAULT_OEF if TYPE_CHECKING: - from packages.skills.weather_station.weather_station_data_model import WEATHER_STATION_DATAMODEL, SCHEME, SERVICE_ID + from packages.skills.weather_station.strategy import Strategy else: - from weather_station_skill.weather_station_data_model import WEATHER_STATION_DATAMODEL, SCHEME, SERVICE_ID + from weather_station_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_station_skill") +logger = logging.getLogger("aea.weather_station_ledger_skill") -REGISTER_ID = 1 +SERVICE_ID = '' -class MyWeatherBehaviour(Behaviour): - """This class scaffolds a behaviour.""" +class ServiceRegistrationBehaviour(Behaviour): + """This class implements a behaviour.""" def __init__(self, **kwargs): """Initialise the behaviour.""" super().__init__(**kwargs) - self.registered = False - self.data_model = WEATHER_STATION_DATAMODEL() - self.scheme = SCHEME + self._registered = False def setup(self) -> None: """ @@ -53,27 +52,42 @@ def setup(self) -> None: :return: None """ - pass + if self.context.ledger_apis.has_fetchai: + fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + if fet_balance > 0: + logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) + else: + logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) - def act(self) -> None: - """ - Implement the act. + if self.context.ledger_apis.has_ethereum: + eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + if eth_balance > 0: + logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) + else: + logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) - :return: None - """ - if not self.registered: - desc = Description(self.scheme, data_model=self.data_model) + if not self._registered: + strategy = cast(Strategy, self.context.strategy) + desc = strategy.get_service_description() + oef_msg_id = strategy.get_next_oef_msg_id() msg = OEFMessage(oef_type=OEFMessage.Type.REGISTER_SERVICE, - id=REGISTER_ID, + id=oef_msg_id, service_description=desc, service_id=SERVICE_ID) - msg_bytes = OEFSerializer().encode(msg) self.context.outbox.put_message(to=DEFAULT_OEF, sender=self.context.agent_public_key, protocol_id=OEFMessage.protocol_id, - message=msg_bytes) - logger.info("[{}]: registered! My public key is : {}".format(self.context.agent_name, self.context.agent_public_key)) - self.registered = True + message=OEFSerializer().encode(msg)) + logger.info("[{}]: registering weather station services on OEF.".format(self.context.agent_name)) + self._registered = True + + def act(self) -> None: + """ + Implement the act. + + :return: None + """ + pass def teardown(self) -> None: """ @@ -81,4 +95,25 @@ def teardown(self) -> None: :return: None """ - pass + if self.context.ledger_apis.has_fetchai: + balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) + logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) + + if self.context.ledger_apis.has_ethereum: + balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) + logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) + + if self._registered: + strategy = cast(Strategy, self.context.strategy) + desc = strategy.get_service_description() + oef_msg_id = strategy.get_next_oef_msg_id() + msg = OEFMessage(oef_type=OEFMessage.Type.UNREGISTER_SERVICE, + id=oef_msg_id, + service_description=desc, + service_id=SERVICE_ID) + self.context.outbox.put_message(to=DEFAULT_OEF, + sender=self.context.agent_public_key, + protocol_id=OEFMessage.protocol_id, + message=OEFSerializer().encode(msg)) + logger.info("[{}]: unregistering weather station services from OEF.".format(self.context.agent_name)) + self._registered = False diff --git a/packages/skills/weather_station/dialogues.py b/packages/skills/weather_station/dialogues.py new file mode 100644 index 0000000000..f844088610 --- /dev/null +++ b/packages/skills/weather_station/dialogues.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- + +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +""" +This module contains the classes required for dialogue management. + +- DialogueLabel: The dialogue label class acts as an identifier for dialogues. +- Dialogue: The dialogue class maintains state of a dialogue and manages it. +- Dialogues: The dialogues class keeps track of all dialogues. +""" + +from enum import Enum +import logging +from typing import Any, Dict, Optional, cast + +from aea.helpers.dialogue.base import DialogueLabel +from aea.helpers.dialogue.base import Dialogue as BaseDialogue +from aea.mail.base import Address +from aea.protocols.base import Message +from aea.protocols.fipa.message import FIPAMessage, VALID_PREVIOUS_PERFORMATIVES +from aea.protocols.oef.models import Description +from aea.skills.base import SharedClass + +logger = logging.getLogger("aea.weather_station_ledger_skill") + + +class Dialogue(BaseDialogue): + """The dialogue class maintains state of a dialogue and manages it.""" + + class EndState(Enum): + """This class defines the end states of a dialogue.""" + + SUCCESSFUL = 0 + DECLINED_PROPOSE = 1 + + def __init__(self, dialogue_label: DialogueLabel, **kwargs) -> None: + """ + Initialize a dialogue label. + + :param dialogue_label: the identifier of the dialogue + + :return: None + """ + BaseDialogue.__init__(self, dialogue_label=dialogue_label) + self.weather_data = None # type: Optional[Dict[str, Any]] + self.proposal = None # type: Optional[Description] + + def is_valid_next_message(self, fipa_msg: FIPAMessage) -> bool: + """ + Check whether this is a valid next message in the dialogue. + + :return: True if yes, False otherwise. + """ + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = cast(FIPAMessage.Performative, fipa_msg.get("performative")) + last_outgoing_message = self.last_outgoing_message + if last_outgoing_message is None: + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP + else: + last_message_id = cast(int, last_outgoing_message.get("message_id")) + last_target = cast(int, last_outgoing_message.get("target")) + last_performative = cast(FIPAMessage.Performative, last_outgoing_message.get("performative")) + result = this_message_id == last_message_id + 1 and \ + this_target == last_target + 1 and \ + last_performative in VALID_PREVIOUS_PERFORMATIVES[this_performative] + return result + + +class DialogueStats(object): + """Class to handle statistics on the negotiation.""" + + def __init__(self) -> None: + """Initialize a StatsManager.""" + self._other_initiated = {Dialogue.EndState.SUCCESSFUL: 0, + Dialogue.EndState.DECLINED_PROPOSE: 0} # type: Dict[Dialogue.EndState, int] + + @property + def other_initiated(self) -> Dict[Dialogue.EndState, int]: + """Get the stats dictionary on other initiated dialogues.""" + return self._other_initiated + + def add_dialogue_endstate(self, end_state: Dialogue.EndState) -> None: + """ + Add dialogue endstate stats. + + :param end_state: the end state of the dialogue + + :return: None + """ + self._other_initiated[end_state] += 1 + + +class Dialogues(SharedClass): + """The dialogues class keeps track of all dialogues.""" + + def __init__(self, **kwargs) -> None: + """ + Initialize dialogues. + + :return: None + """ + SharedClass.__init__(self, **kwargs) + self._dialogues = {} # type: Dict[DialogueLabel, Dialogue] + self._dialogue_stats = DialogueStats() + + @property + def dialogues(self) -> Dict[DialogueLabel, Dialogue]: + """Get dictionary of dialogues in which the agent is engaged in.""" + return self._dialogues + + @property + def dialogue_stats(self) -> DialogueStats: + """Get the dialogue statistics.""" + return self._dialogue_stats + + def is_permitted_for_new_dialogue(self, fipa_msg: Message, sender: Address) -> bool: + """ + Check whether a fipa message is permitted for a new dialogue. + + That is, the message has to + - be a CFP, and + - have the correct msg id and message target. + + :param message: the fipa message + :param sender: the sender + + :return: a boolean indicating whether the message is permitted for a new dialogue + """ + fipa_msg = cast(FIPAMessage, fipa_msg) + this_message_id = fipa_msg.get("message_id") + this_target = fipa_msg.get("target") + this_performative = fipa_msg.get("performative") + + result = this_message_id == FIPAMessage.STARTING_MESSAGE_ID and \ + this_target == FIPAMessage.STARTING_TARGET and \ + this_performative == FIPAMessage.Performative.CFP + return result + + def is_belonging_to_registered_dialogue(self, fipa_msg: Message, sender: Address, agent_pbk: Address) -> bool: + """ + Check whether an agent message is part of a registered dialogue. + + :param fipa_msg: the fipa message + :param sender: the sender + :param agent_pbk: the public key of the agent + + :return: boolean indicating whether the message belongs to a registered dialogue + """ + fipa_msg = cast(FIPAMessage, fipa_msg) + dialogue_id = cast(int, fipa_msg.get("dialogue_id")) + other_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, sender) + if other_initiated_dialogue_label in self.dialogues: + other_initiated_dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) + result = other_initiated_dialogue.is_valid_next_message(fipa_msg) + else: + result = False + return result + + def get_dialogue(self, dialogue_id: int, sender: Address, agent_pbk: Address) -> Dialogue: + """ + Retrieve dialogue. + + :param dialogue_id: the dialogue id + :param sender_pbk: the sender public key + :param agent_pbk: the public key of the agent + + :return: the dialogue + """ + other_initiated_dialogue_label = DialogueLabel(dialogue_id, sender, sender) + dialogue = cast(Dialogue, self.dialogues[other_initiated_dialogue_label]) + return dialogue + + def create_opponent_initiated(self, dialogue_id: int, sender: Address) -> Dialogue: + """ + Save an opponent initiated dialogue. + + :param dialogue_id: the dialogue id + :param sender: the pbk of the sender + + :return: the created dialogue + """ + dialogue_starter_pbk = sender + dialogue_opponent_pbk = sender + dialogue_label = DialogueLabel(dialogue_id, dialogue_opponent_pbk, dialogue_starter_pbk) + result = self._create(dialogue_label) + return result + + def _create(self, dialogue_label: DialogueLabel) -> Dialogue: + """ + Create a dialogue. + + :param dialogue_label: the dialogue label + + :return: the created dialogue + """ + assert dialogue_label not in self.dialogues + dialogue = Dialogue(dialogue_label) + self.dialogues.update({dialogue_label: dialogue}) + return dialogue + + def reset(self) -> None: + """ + Reset the dialogues. + + :return: None + """ + self._dialogues = {} + self._dialogue_stats = DialogueStats() diff --git a/packages/skills/weather_station/dummy_weather_station_data.py b/packages/skills/weather_station/dummy_weather_station_data.py index 126b091e3d..43054a533e 100644 --- a/packages/skills/weather_station/dummy_weather_station_data.py +++ b/packages/skills/weather_station/dummy_weather_station_data.py @@ -27,7 +27,7 @@ import time from typing import Dict, Union -logger = logging.getLogger("aea.weather_station_skill") +logger = logging.getLogger("aea.weather_station_ledger_skill") my_path = os.path.dirname(__file__) @@ -61,7 +61,7 @@ cur.close() con.commit() if con is not None: - logger.info("Wheather station: I closed the db after checking it is populated!") + logger.debug("Weather station: I closed the db after checking it is populated!") con.close() diff --git a/packages/skills/weather_station/handlers.py b/packages/skills/weather_station/handlers.py index 4c2dd95c9a..88f481f3f4 100644 --- a/packages/skills/weather_station/handlers.py +++ b/packages/skills/weather_station/handlers.py @@ -20,9 +20,7 @@ """This package contains a scaffold of a handler.""" import logging -import json -import time -from typing import Any, Dict, List, Optional, Union, cast, TYPE_CHECKING +from typing import Optional, cast, TYPE_CHECKING from aea.configurations.base import ProtocolId from aea.protocols.base import Message @@ -30,53 +28,63 @@ from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer -from aea.protocols.oef.models import Description +from aea.protocols.oef.models import Description, Query from aea.skills.base import Handler if TYPE_CHECKING: - from packages.skills.weather_station.db_communication import DBCommunication + from packages.skills.weather_station.dialogues import Dialogue, Dialogues + from packages.skills.weather_station.strategy import Strategy else: - from weather_station_skill.db_communication import DBCommunication + from weather_station_skill.dialogues import Dialogue, Dialogues + from weather_station_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_station_skill") +logger = logging.getLogger("aea.weather_station_ledger_skill") -DATE_ONE = "3/10/2019" -DATE_TWO = "15/10/2019" - -class MyWeatherHandler(Handler): +class FIPAHandler(Handler): """This class scaffolds a handler.""" SUPPORTED_PROTOCOL = FIPAMessage.protocol_id # type: Optional[ProtocolId] - def __init__(self, **kwargs): - """Initialise the behaviour.""" - super().__init__(**kwargs) - self.fet_price = 0.002 - self.db = DBCommunication() - self.fetched_data = [] - def setup(self) -> None: """Implement the setup for the handler.""" pass def handle(self, message: Message, sender: str) -> None: """ - Implement the reaction to an message. + Implement the reaction to a message. :param message: the message :param sender: the sender :return: None """ + # convenience representations fipa_msg = cast(FIPAMessage, message) msg_performative = FIPAMessage.Performative(fipa_msg.get('performative')) message_id = cast(int, fipa_msg.get('message_id')) dialogue_id = cast(int, fipa_msg.get('dialogue_id')) + # recover dialogue + dialogues = cast(Dialogues, self.context.dialogues) + if dialogues.is_belonging_to_registered_dialogue(fipa_msg, sender, self.context.agent_public_key): + dialogue = dialogues.get_dialogue(dialogue_id, sender, self.context.agent_public_key) + dialogue.incoming_extend(fipa_msg) + elif dialogues.is_permitted_for_new_dialogue(fipa_msg, sender): + dialogue = dialogues.create_opponent_initiated(dialogue_id, sender) + dialogue.incoming_extend(fipa_msg) + else: + self._handle_unidentified_dialogue(fipa_msg, sender) + return + + # handle message if msg_performative == FIPAMessage.Performative.CFP: - self.handle_cfp(fipa_msg, sender, message_id, dialogue_id) + self._handle_cfp(fipa_msg, sender, message_id, dialogue_id, dialogue) + elif msg_performative == FIPAMessage.Performative.DECLINE: + self._handle_decline(fipa_msg, sender, message_id, dialogue_id, dialogue) elif msg_performative == FIPAMessage.Performative.ACCEPT: - self.handle_accept(sender) + self._handle_accept(fipa_msg, sender, message_id, dialogue_id, dialogue) + elif msg_performative == FIPAMessage.Performative.INFORM: + self._handle_inform(fipa_msg, sender, message_id, dialogue_id, dialogue) def teardown(self) -> None: """ @@ -86,81 +94,161 @@ def teardown(self) -> None: """ pass - def handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int) -> None: + def _handle_unidentified_dialogue(self, msg: FIPAMessage, sender: str) -> None: + """ + Handle an unidentified dialogue. + + Respond to the sender with a default message containing the appropriate error information. + + :param msg: the message + :param sender: the sender + :return: None + """ + logger.info("[{}]: unidentified dialogue.".format(self.context.agent_name)) + default_msg = DefaultMessage(type=DefaultMessage.Type.ERROR, + error_code=DefaultMessage.ErrorCode.INVALID_DIALOGUE.value, + error_msg="Invalid dialogue.", + error_data="fipa_message") # FIPASerializer().encode(msg) + self.context.outbox.put_message(to=sender, + sender=self.context.agent_public_key, + protocol_id=DefaultMessage.protocol_id, + message=DefaultSerializer().encode(default_msg)) + + def _handle_cfp(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ - Handle the CFP calls. + Handle the CFP. + + If the CFP matches the supplied services then send a PROPOSE, otherwise send a DECLINE. :param msg: the message :param sender: the sender :param message_id: the message id :param dialogue_id: the dialogue id + :param dialogue: the dialogue object :return: None """ new_message_id = message_id + 1 new_target = message_id - fetched_data = self.db.get_data_for_specific_dates(DATE_ONE, DATE_TWO) - - if len(fetched_data) >= 1: - self.fetched_data = fetched_data - total_price = self.fet_price * len(fetched_data) - proposal = [Description({"Rows": len(fetched_data), - "Price": total_price})] - logger.info("[{}]: sending sender={} a proposal at price={}".format(self.context.agent_name, sender, total_price)) + logger.info("[{}]: received CFP from sender={}".format(self.context.agent_name, + sender[-5:])) + query = cast(Query, msg.get("query")) + strategy = cast(Strategy, self.context.strategy) + + if strategy.is_matching_supply(query): + proposal, weather_data = strategy.generate_proposal_and_data(query) + dialogue.weather_data = weather_data + dialogue.proposal = proposal + logger.info("[{}]: sending sender={} a PROPOSE with proposal={}".format(self.context.agent_name, + sender[-5:], + proposal.values)) proposal_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target, performative=FIPAMessage.Performative.PROPOSE, - proposal=proposal) + proposal=[proposal]) + dialogue.outgoing_extend(proposal_msg) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(proposal_msg)) else: - logger.info("[{}]: declined the CFP from sender={}".format(self.context.agent_name, sender)) + logger.info("[{}]: declined the CFP from sender={}".format(self.context.agent_name, + sender[-5:])) decline_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target, performative=FIPAMessage.Performative.DECLINE) + dialogue.outgoing_extend(decline_msg) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(decline_msg)) - def handle_accept(self, sender: str) -> None: + def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ - Handle the Accept Calls. + Handle the DECLINE. + + Close the dialogue. + :param msg: the message :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object :return: None """ - command = {} # type: Dict[str, Union[str, List[Any]]] - command['Command'] = "success" - command['fetched_data'] = [] - counter = 0 - for items in self.fetched_data: - dict_of_data = { - 'abs_pressure': items[0], - 'delay': items[1], - 'hum_in': items[2], - 'hum_out': items[3], - 'idx': time.ctime(int(items[4])), - 'rain': items[5], - 'temp_in': items[6], - 'temp_out': items[7], - 'wind_ave': items[8], - 'wind_dir': items[9], - 'wind_gust': items[10] - } - command['fetched_data'].append(dict_of_data) # type: ignore - counter += 1 - if counter == 10: - break - json_data = json.dumps(command) - json_bytes = json_data.encode("utf-8") - logger.info("[{}]: handling accept and sending weather data to sender={}".format(self.context.agent_name, sender)) - data_msg = DefaultMessage( - type=DefaultMessage.Type.BYTES, content=json_bytes) + logger.info("[{}]: received DECLINE from sender={}".format(self.context.agent_name, + sender[-5:])) + # dialogues = cast(Dialogues, self.context.dialogues) + # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.DECLINED_PROPOSE) + + def _handle_accept(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, + dialogue: Dialogue) -> None: + """ + Handle the ACCEPT. + + Respond with a MATCH_ACCEPT_W_ADDRESS which contains the address to send the funds to. + + :param msg: the message + :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object + :return: None + """ + new_message_id = message_id + 1 + new_target = message_id + logger.info("[{}]: received ACCEPT from sender={}".format(self.context.agent_name, + sender[-5:])) + logger.info("[{}]: sending MATCH_ACCEPT_W_ADDRESS to sender={}".format(self.context.agent_name, + sender[-5:])) + proposal = cast(Description, dialogue.proposal) + identifier = cast(str, proposal.values.get("ledger_id")) + match_accept_msg = FIPAMessage(message_id=new_message_id, + dialogue_id=dialogue_id, + target=new_target, + performative=FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS, + address=self.context.agent_addresses[identifier]) + dialogue.outgoing_extend(match_accept_msg) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, - protocol_id=DefaultMessage.protocol_id, - message=DefaultSerializer().encode(data_msg)) + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(match_accept_msg)) + + def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, + dialogue: Dialogue) -> None: + """ + Handle the INFORM. + + If the INFORM message contains the transaction_digest then verify that it is settled, otherwise do nothing. + If the transaction is settled send the weather data, otherwise do nothing. + + :param msg: the message + :param sender: the sender + :param message_id: the message id + :param dialogue_id: the dialogue id + :param dialogue: the dialogue object + :return: None + """ + new_message_id = message_id + 1 + new_target = message_id + logger.info("[{}]: received INFORM from sender={}".format(self.context.agent_name, + sender[-5:])) + + json_data = cast(dict, msg.get("json_data")) + if "transaction" in json_data: + inform_msg = FIPAMessage(message_id=new_message_id, + dialogue_id=dialogue_id, + target=new_target, + performative=FIPAMessage.Performative.INFORM, + json_data=dialogue.weather_data) + dialogue.outgoing_extend(inform_msg) + # import pdb; pdb.set_trace() + self.context.outbox.put_message(to=sender, + sender=self.context.agent_public_key, + protocol_id=FIPAMessage.protocol_id, + message=FIPASerializer().encode(inform_msg)) + # dialogues = cast(Dialogues, self.context.dialogues) + # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.SUCCESSFUL) + else: + logger.warning("I didn't receive the transaction digest!") diff --git a/packages/skills/weather_station/skill.yaml b/packages/skills/weather_station/skill.yaml index c817a83767..e9fa57817c 100644 --- a/packages/skills/weather_station/skill.yaml +++ b/packages/skills/weather_station/skill.yaml @@ -3,17 +3,27 @@ authors: Fetch.AI Limited version: 0.1.0 license: Apache 2.0 url: "" -description: "The weather station skill offers weather data for sale." behaviours: - behaviour: - class_name: MyWeatherBehaviour - args: - foo: bar + class_name: ServiceRegistrationBehaviour + args: {} handlers: - handler: - class_name: MyWeatherHandler - args: - foo: bar + class_name: FIPAHandler + args: {} tasks: [] -shared_classes: [] -protocols: ['fipa'] +shared_classes: + - shared_class: + class_name: Strategy + args: + date_one: "1/10/2019" + date_two: "1/12/2019" + price_per_row: 1 + seller_tx_fee: 0 + currency_pbk: 'FET' + ledger_id: 'fetchai' + - shared_class: + class_name: Dialogues + args: {} +protocols: ['fipa', 'oef', 'default'] +ledgers: ['fetchai'] diff --git a/packages/skills/weather_station/strategy.py b/packages/skills/weather_station/strategy.py new file mode 100644 index 0000000000..4391d44453 --- /dev/null +++ b/packages/skills/weather_station/strategy.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the strategy class.""" +import time +from typing import Any, Dict, List, Tuple, TYPE_CHECKING + +from aea.protocols.oef.models import Description, Query +from aea.skills.base import SharedClass + +if TYPE_CHECKING: + from packages.skills.weather_station.db_communication import DBCommunication + from packages.skills.weather_station.weather_station_data_model import WEATHER_STATION_DATAMODEL, SCHEME +else: + from weather_station_skill.db_communication import DBCommunication + from weather_station_skill.weather_station_data_model import WEATHER_STATION_DATAMODEL, SCHEME + +DEFAULT_PRICE_PER_ROW = 2 +DEFAULT_SELLER_TX_FEE = 0 +DEFAULT_CURRENCY_PBK = 'FET' +DEFAULT_LEDGER_ID = 'fetchai' +DEFAULT_DATE_ONE = "3/10/2019" +DEFAULT_DATE_TWO = "15/10/2019" + + +class Strategy(SharedClass): + """This class defines a strategy for the agent.""" + + def __init__(self, **kwargs) -> None: + """ + Initialize the strategy of the agent. + + :param register_as: determines whether the agent registers as seller, buyer or both + :param search_for: determines whether the agent searches for sellers, buyers or both + + :return: None + """ + self._price_per_row = kwargs.pop('price_per_row') if 'price_per_row' in kwargs.keys() else DEFAULT_PRICE_PER_ROW + self._seller_tx_fee = kwargs.pop('seller_tx_fee') if 'seller_tx_fee' in kwargs.keys() else DEFAULT_SELLER_TX_FEE + self._currency_pbk = kwargs.pop('currency_pbk') if 'currency_pbk' in kwargs.keys() else DEFAULT_CURRENCY_PBK + self._ledger_id = kwargs.pop('ledger_id') if 'ledger_id' in kwargs.keys() else DEFAULT_LEDGER_ID + self._date_one = kwargs.pop('date_one') if 'date_one' in kwargs.keys() else DEFAULT_DATE_ONE + self._date_two = kwargs.pop('date_two') if 'date_two' in kwargs.keys() else DEFAULT_DATE_TWO + super().__init__(**kwargs) + self.db = DBCommunication() + self._oef_msg_id = 0 + + def get_next_oef_msg_id(self) -> int: + """ + Get the next oef msg id. + + :return: the next oef msg id + """ + self._oef_msg_id += 1 + return self._oef_msg_id + + def get_service_description(self) -> Description: + """ + Get the service description. + + :return: a description of the offered services + """ + desc = Description(SCHEME, data_model=WEATHER_STATION_DATAMODEL()) + return desc + + def is_matching_supply(self, query: Query) -> bool: + """ + Check if the query matches the supply. + + :param query: the query + :return: bool indiciating whether matches or not + """ + # TODO, this is a stub + return True + + def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[str, List[Dict[str, Any]]]]: + """ + Generate a proposal matching the query. + + :param query: the query + :return: a tuple of proposal and the weather data + """ + fetched_data = self.db.get_data_for_specific_dates(self._date_one, self._date_two) # TODO: fetch real data + weather_data, rows = self._build_data_payload(fetched_data) + total_price = self._price_per_row * rows + assert total_price - self._seller_tx_fee > 0, "This sale would generate a loss, change the configs!" + proposal = Description({"rows": rows, + "price": total_price, + "seller_tx_fee": self._seller_tx_fee, + "currency_pbk": self._currency_pbk, + "ledger_id": self._ledger_id}) + return (proposal, weather_data) + + def _build_data_payload(self, fetched_data: Dict[str, int]) -> Tuple[Dict[str, List[Dict[str, Any]]], int]: + """ + Build the data payload. + + :param fetched_data: the fetched data + :return: a tuple of the data and the rows + """ + weather_data = {} # type: Dict[str, List[Dict[str, Any]]] + weather_data['weather_data'] = [] + counter = 0 + for items in fetched_data: + if counter > 10: + break # TODO: fix OEF so more data can be sent + counter += 1 + dict_of_data = { + 'abs_pressure': items[0], + 'delay': items[1], + 'hum_in': items[2], + 'hum_out': items[3], + 'idx': time.ctime(int(items[4])), + 'rain': items[5], + 'temp_in': items[6], + 'temp_out': items[7], + 'wind_ave': items[8], + 'wind_dir': items[9], + 'wind_gust': items[10] + } + weather_data['weather_data'].append(dict_of_data) + return weather_data, counter diff --git a/packages/skills/weather_station/weather_station_data_model.py b/packages/skills/weather_station/weather_station_data_model.py index 43783bb541..778de82e95 100644 --- a/packages/skills/weather_station/weather_station_data_model.py +++ b/packages/skills/weather_station/weather_station_data_model.py @@ -22,7 +22,6 @@ from aea.protocols.oef.models import DataModel, Attribute SCHEME = {'country': "UK", 'city': "Cambridge"} -SERVICE_ID = "WeatherData" class WEATHER_STATION_DATAMODEL (DataModel): From 136c84743681985bbec494ddbdb8f87ba2e99ab7 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 13:48:07 +0000 Subject: [PATCH 16/38] Cleaned unused code based on PR comments --- packages/skills/weather_client/behaviours.py | 26 ++----------------- packages/skills/weather_client/handlers.py | 2 +- packages/skills/weather_client/skill.yaml | 4 +-- packages/skills/weather_station/behaviours.py | 24 ----------------- .../dummy_weather_station_data.py | 2 +- packages/skills/weather_station/handlers.py | 10 +++---- packages/skills/weather_station/skill.yaml | 4 +-- 7 files changed, 13 insertions(+), 59 deletions(-) diff --git a/packages/skills/weather_client/behaviours.py b/packages/skills/weather_client/behaviours.py index 37f876ec00..818cdf0618 100644 --- a/packages/skills/weather_client/behaviours.py +++ b/packages/skills/weather_client/behaviours.py @@ -21,8 +21,6 @@ import logging from typing import cast, TYPE_CHECKING -from aea.crypto.ethereum import ETHEREUM -from aea.crypto.fetchai import FETCHAI from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.serialization import DEFAULT_OEF, OEFSerializer from aea.skills.base import Behaviour @@ -44,21 +42,7 @@ def __init__(self, **kwargs): def setup(self) -> None: """Implement the setup for the behaviour.""" - if self.context.ledger_apis.has_fetchai: - fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) - if fet_balance > 0: - logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) - else: - logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) - # TODO: deregister skill from filter - - if self.context.ledger_apis.has_ethereum: - eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) - if eth_balance > 0: - logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) - else: - logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) - # TODO: deregister skill from filter + pass def act(self) -> None: """ @@ -84,10 +68,4 @@ def teardown(self) -> None: :return: None """ - if self.context.ledger_apis.has_fetchai: - balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) - logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) - - if self.context.ledger_apis.has_ethereum: - balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) - logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) + pass diff --git a/packages/skills/weather_client/handlers.py b/packages/skills/weather_client/handlers.py index 5004f61bba..1d05b4171b 100644 --- a/packages/skills/weather_client/handlers.py +++ b/packages/skills/weather_client/handlers.py @@ -39,7 +39,7 @@ from weather_client_skill.dialogues import Dialogue, Dialogues from weather_client_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_client_ledger_skill") +logger = logging.getLogger("aea.weather_client_skill") class FIPAHandler(Handler): diff --git a/packages/skills/weather_client/skill.yaml b/packages/skills/weather_client/skill.yaml index 7aea8fba33..aeeba7ddf9 100644 --- a/packages/skills/weather_client/skill.yaml +++ b/packages/skills/weather_client/skill.yaml @@ -24,9 +24,9 @@ shared_classes: max_row_price: 4 max_buyer_tx_fee: 1 currency_pbk: 'FET' - ledger_id: 'fetchai' + ledger_id: 'None' - shared_class: class_name: Dialogues args: {} protocols: ['fipa','default','oef'] -ledgers: ['fetchai'] +ledgers: [] diff --git a/packages/skills/weather_station/behaviours.py b/packages/skills/weather_station/behaviours.py index 3b0da66eff..1dcbb5a21a 100644 --- a/packages/skills/weather_station/behaviours.py +++ b/packages/skills/weather_station/behaviours.py @@ -22,8 +22,6 @@ import logging from typing import cast, TYPE_CHECKING -from aea.crypto.ethereum import ETHEREUM -from aea.crypto.fetchai import FETCHAI from aea.skills.base import Behaviour from aea.protocols.oef.message import OEFMessage from aea.protocols.oef.serialization import OEFSerializer, DEFAULT_OEF @@ -52,20 +50,6 @@ def setup(self) -> None: :return: None """ - if self.context.ledger_apis.has_fetchai: - fet_balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) - if fet_balance > 0: - logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, fet_balance)) - else: - logger.warning("[{}]: you have no starting balance on fetchai ledger!".format(self.context.agent_name)) - - if self.context.ledger_apis.has_ethereum: - eth_balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) - if eth_balance > 0: - logger.info("[{}]: starting balance on ethereum ledger={}.".format(self.context.agent_name, eth_balance)) - else: - logger.warning("[{}]: you have no starting balance on ethereum ledger!".format(self.context.agent_name)) - if not self._registered: strategy = cast(Strategy, self.context.strategy) desc = strategy.get_service_description() @@ -95,14 +79,6 @@ def teardown(self) -> None: :return: None """ - if self.context.ledger_apis.has_fetchai: - balance = self.context.ledger_apis.token_balance(FETCHAI, cast(str, self.context.agent_addresses.get(FETCHAI))) - logger.info("[{}]: ending balance on fetchai ledger={}.".format(self.context.agent_name, balance)) - - if self.context.ledger_apis.has_ethereum: - balance = self.context.ledger_apis.token_balance(ETHEREUM, cast(str, self.context.agent_addresses.get(ETHEREUM))) - logger.info("[{}]: ending balance on ethereum ledger={}.".format(self.context.agent_name, balance)) - if self._registered: strategy = cast(Strategy, self.context.strategy) desc = strategy.get_service_description() diff --git a/packages/skills/weather_station/dummy_weather_station_data.py b/packages/skills/weather_station/dummy_weather_station_data.py index 43054a533e..5d4ca6274c 100644 --- a/packages/skills/weather_station/dummy_weather_station_data.py +++ b/packages/skills/weather_station/dummy_weather_station_data.py @@ -27,7 +27,7 @@ import time from typing import Dict, Union -logger = logging.getLogger("aea.weather_station_ledger_skill") +logger = logging.getLogger("aea.weather_station_skill") my_path = os.path.dirname(__file__) diff --git a/packages/skills/weather_station/handlers.py b/packages/skills/weather_station/handlers.py index 88f481f3f4..558aa96678 100644 --- a/packages/skills/weather_station/handlers.py +++ b/packages/skills/weather_station/handlers.py @@ -28,7 +28,7 @@ from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer -from aea.protocols.oef.models import Description, Query +from aea.protocols.oef.models import Query # Description from aea.skills.base import Handler if TYPE_CHECKING: @@ -38,7 +38,7 @@ from weather_station_skill.dialogues import Dialogue, Dialogues from weather_station_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_station_ledger_skill") +logger = logging.getLogger("aea.weather_station_skill") class FIPAHandler(Handler): @@ -202,13 +202,13 @@ def _handle_accept(self, msg: FIPAMessage, sender: str, message_id: int, dialogu sender[-5:])) logger.info("[{}]: sending MATCH_ACCEPT_W_ADDRESS to sender={}".format(self.context.agent_name, sender[-5:])) - proposal = cast(Description, dialogue.proposal) - identifier = cast(str, proposal.values.get("ledger_id")) + # proposal = cast(Description, dialogue.proposal) + # identifier = cast(str, proposal.values.get("ledger_id")) match_accept_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target, performative=FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS, - address=self.context.agent_addresses[identifier]) + address="no_address") dialogue.outgoing_extend(match_accept_msg) self.context.outbox.put_message(to=sender, sender=self.context.agent_public_key, diff --git a/packages/skills/weather_station/skill.yaml b/packages/skills/weather_station/skill.yaml index e9fa57817c..3141455150 100644 --- a/packages/skills/weather_station/skill.yaml +++ b/packages/skills/weather_station/skill.yaml @@ -21,9 +21,9 @@ shared_classes: price_per_row: 1 seller_tx_fee: 0 currency_pbk: 'FET' - ledger_id: 'fetchai' + ledger_id: 'None' - shared_class: class_name: Dialogues args: {} protocols: ['fipa', 'oef', 'default'] -ledgers: ['fetchai'] +ledgers: [] From d489c15d901135778b11a600b12c87f2bbc814c2 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 13:53:57 +0000 Subject: [PATCH 17/38] Changed based on PR Comments --- packages/skills/weather_client/strategy.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/skills/weather_client/strategy.py b/packages/skills/weather_client/strategy.py index 60fa0f3279..3fc61b5af8 100644 --- a/packages/skills/weather_client/strategy.py +++ b/packages/skills/weather_client/strategy.py @@ -31,7 +31,7 @@ DEFAULT_MAX_ROW_PRICE = 5 DEFAULT_MAX_TX_FEE = 2 DEFAULT_CURRENCY_PBK = 'FET' -DEFAULT_LEDGER_ID = 'fetchai' +DEFAULT_LEDGER_ID = 'None' class Strategy(SharedClass): @@ -97,15 +97,3 @@ def is_acceptable_proposal(self, proposal: Description) -> bool: (proposal.values['currency_pbk'] == self._currency_pbk) and \ (proposal.values['ledger_id'] == self._ledger_id) return result - - def is_affordable_proposal(self, proposal: Description) -> bool: - """ - Check whether it is an affordable proposal. - - :return: whether it is affordable - """ - payable = proposal.values['price'] + self.max_buyer_tx_fee - ledger_id = proposal.values['ledger_id'] - address = cast(str, self.context.agent_addresses.get(ledger_id)) - balance = self.context.ledger_apis.token_balance(ledger_id, address) - return balance >= payable From 25f57a47025d07da7aa388b8f2c75911f2369311 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Nov 2019 14:24:48 +0000 Subject: [PATCH 18/38] Rearrange modules and functions to avoid circular imports and clean up --- aea/cli/run.py | 99 +++++++++- aea/crypto/default.py | 9 + aea/crypto/helpers.py | 174 ++++-------------- aea/crypto/ledger_apis.py | 32 ++++ aea/crypto/wallet.py | 2 - packages/skills/carpark_detection/handlers.py | 2 +- tests/test_crypto/test_helpers.py | 2 +- 7 files changed, 170 insertions(+), 150 deletions(-) diff --git a/aea/cli/run.py b/aea/cli/run.py index 13dbdb9a90..cd19680744 100644 --- a/aea/cli/run.py +++ b/aea/cli/run.py @@ -34,12 +34,103 @@ AEAConfigException, _load_env_file from aea.cli.install import install from aea.connections.base import Connection -from aea.crypto.helpers import _verify_or_create_private_keys, _verify_ledger_apis_access -from aea.crypto.ledger_apis import LedgerApis -from aea.crypto.wallet import Wallet, DEFAULT +from aea.configurations.loader import ConfigLoader +from aea.configurations.base import AgentConfig, DEFAULT_AEA_CONFIG_FILE, PrivateKeyPathConfig, LedgerAPIConfig +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI +from aea.crypto.helpers import _create_default_private_key, _create_fetchai_private_key, _create_ethereum_private_key, DEFAULT_PRIVATE_KEY_FILE, FETCHAI_PRIVATE_KEY_FILE, ETHEREUM_PRIVATE_KEY_FILE, _try_validate_private_key_pem_path, _try_validate_fet_private_key_path, _try_validate_ethereum_private_key_path +from aea.crypto.ledger_apis import LedgerApis, _try_to_instantiate_fetchai_ledger_api, _try_to_instantiate_ethereum_ledger_api, SUPPORTED_LEDGER_APIS +from aea.crypto.wallet import Wallet, DEFAULT, SUPPORTED_CRYPTOS from aea.mail.base import MailBox +def _verify_or_create_private_keys(ctx: Context) -> None: + """ + Verify or create private keys. + + :param ctx: Context + """ + path = Path(DEFAULT_AEA_CONFIG_FILE) + agent_loader = ConfigLoader("aea-config_schema.json", AgentConfig) + fp = open(str(path), mode="r", encoding="utf-8") + aea_conf = agent_loader.load(fp) + + for identifier, value in aea_conf.private_key_paths.read_all(): + if identifier not in SUPPORTED_CRYPTOS: + ValueError("Unsupported identifier in private key paths.") + + default_private_key_config = aea_conf.private_key_paths.read(DEFAULT) + if default_private_key_config is None: + _create_default_private_key() + default_private_key_config = PrivateKeyPathConfig(DEFAULT, DEFAULT_PRIVATE_KEY_FILE) + aea_conf.private_key_paths.create(default_private_key_config.ledger, default_private_key_config) + else: + default_private_key_config = cast(PrivateKeyPathConfig, default_private_key_config) + try: + _try_validate_private_key_pem_path(default_private_key_config.path) + except FileNotFoundError: + logger.error("File {} for private key {} not found.".format(repr(default_private_key_config.path), default_private_key_config.ledger)) + sys.exit(1) + + fetchai_private_key_config = aea_conf.private_key_paths.read(FETCHAI) + if fetchai_private_key_config is None: + _create_fetchai_private_key() + fetchai_private_key_config = PrivateKeyPathConfig(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) + aea_conf.private_key_paths.create(fetchai_private_key_config.ledger, fetchai_private_key_config) + else: + fetchai_private_key_config = cast(PrivateKeyPathConfig, fetchai_private_key_config) + try: + _try_validate_fet_private_key_path(fetchai_private_key_config.path) + except FileNotFoundError: + logger.error("File {} for private key {} not found.".format(repr(fetchai_private_key_config.path), fetchai_private_key_config.ledger)) + sys.exit(1) + + ethereum_private_key_config = aea_conf.private_key_paths.read(ETHEREUM) + if ethereum_private_key_config is None: + _create_ethereum_private_key() + ethereum_private_key_config = PrivateKeyPathConfig(ETHEREUM, ETHEREUM_PRIVATE_KEY_FILE) + aea_conf.private_key_paths.create(ethereum_private_key_config.ledger, ethereum_private_key_config) + else: + ethereum_private_key_config = cast(PrivateKeyPathConfig, ethereum_private_key_config) + try: + _try_validate_ethereum_private_key_path(ethereum_private_key_config.path) + except FileNotFoundError: + logger.error("File {} for private key {} not found.".format(repr(ethereum_private_key_config.path), ethereum_private_key_config.ledger)) + sys.exit(1) + + # update aea config + path = Path(DEFAULT_AEA_CONFIG_FILE) + fp = open(str(path), mode="w", encoding="utf-8") + agent_loader.dump(aea_conf, fp) + ctx.agent_config = aea_conf + + +def _verify_ledger_apis_access() -> None: + """Verify access to ledger apis.""" + path = Path(DEFAULT_AEA_CONFIG_FILE) + agent_loader = ConfigLoader("aea-config_schema.json", AgentConfig) + fp = open(str(path), mode="r", encoding="utf-8") + aea_conf = agent_loader.load(fp) + + for identifier, value in aea_conf.ledger_apis.read_all(): + if identifier not in SUPPORTED_LEDGER_APIS: + ValueError("Unsupported identifier in ledger apis.") + + fetchai_ledger_api_config = aea_conf.ledger_apis.read(FETCHAI) + if fetchai_ledger_api_config is None: + logger.debug("No fetchai ledger api config specified.") + else: + fetchai_ledger_api_config = cast(LedgerAPIConfig, fetchai_ledger_api_config) + _try_to_instantiate_fetchai_ledger_api(fetchai_ledger_api_config.addr, fetchai_ledger_api_config.port) + + ethereum_ledger_config = aea_conf.ledger_apis.read(ETHEREUM) + if ethereum_ledger_config is None: + logger.debug("No ethereum ledger api config specified.") + else: + ethereum_ledger_config = cast(LedgerAPIConfig, ethereum_ledger_config) + _try_to_instantiate_ethereum_ledger_api(ethereum_ledger_config.addr, ethereum_ledger_config.port) + + def _setup_connection(connection_name: str, public_key: str, ctx: Context) -> Connection: """ Set up a connection. @@ -96,7 +187,7 @@ def run(click_context, connection_name: str, env_file: str, install_deps: bool): agent_name = cast(str, ctx.agent_config.agent_name) _verify_or_create_private_keys(ctx) - _verify_ledger_apis_access(ctx) + _verify_ledger_apis_access() private_key_paths = dict([(identifier, config.path) for identifier, config in ctx.agent_config.private_key_paths.read_all()]) ledger_api_configs = dict([(identifier, (config.addr, config.port)) for identifier, config in ctx.agent_config.ledger_apis.read_all()]) diff --git a/aea/crypto/default.py b/aea/crypto/default.py index ef0f5ef7fe..697485bacd 100644 --- a/aea/crypto/default.py +++ b/aea/crypto/default.py @@ -84,6 +84,15 @@ def public_key_pem(self) -> bytes: """ return self._public_key_pem + @property + def private_key_pem(self) -> bytes: + """ + Return a PEM encoded private key in base64 format. It consists of an algorithm identifier and the private key as a bit string. + + :return: a private key bytes string + """ + return self._private_key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) # type: ignore + @property def address(self) -> str: """ diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index ab15277d64..2581910305 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -20,22 +20,12 @@ """Module wrapping the helpers of public and private key cryptography.""" import sys -from typing import cast - -from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption import logging -from pathlib import Path from fetchai.ledger.crypto import Entity # type: ignore from eth_account import Account # type: ignore -from aea.crypto.default import DefaultCrypto, DEFAULT -from aea.crypto.ethereum import ETHEREUM -from aea.crypto.fetchai import FETCHAI -from aea.crypto.wallet import SUPPORTED_CRYPTOS, SUPPORTED_LEDGER_APIS -from aea.configurations.loader import ConfigLoader -from aea.configurations.base import AgentConfig, DEFAULT_AEA_CONFIG_FILE, PrivateKeyPathConfig, LedgerAPIConfig -from aea.cli.common import Context +from aea.crypto.default import DefaultCrypto DEFAULT_PRIVATE_KEY_FILE = 'default_private_key.pem' FETCHAI_PRIVATE_KEY_FILE = 'fet_private_key.txt' @@ -44,126 +34,6 @@ logger = logging.getLogger(__name__) -def _verify_or_create_private_keys(ctx: Context) -> None: - """ - Verify or create private keys. - - :param ctx: Context - """ - path = Path(DEFAULT_AEA_CONFIG_FILE) - agent_loader = ConfigLoader("aea-config_schema.json", AgentConfig) - fp = open(str(path), mode="r", encoding="utf-8") - aea_conf = agent_loader.load(fp) - - for identifier, value in aea_conf.private_key_paths.read_all(): - if identifier not in SUPPORTED_CRYPTOS: - ValueError("Unsupported identifier in private key paths.") - - default_private_key_config = aea_conf.private_key_paths.read(DEFAULT) - if default_private_key_config is None: - default_private_key_path = _create_temporary_private_key_pem_path() - default_private_key_config = PrivateKeyPathConfig(DEFAULT, default_private_key_path) - aea_conf.private_key_paths.create(default_private_key_config.ledger, default_private_key_config) - else: - default_private_key_config = cast(PrivateKeyPathConfig, default_private_key_config) - try: - _try_validate_private_key_pem_path(default_private_key_config.path) - except FileNotFoundError: - logger.error("File {} for private key {} not found.".format(repr(default_private_key_config.path), default_private_key_config.ledger)) - sys.exit(-1) - - fetchai_private_key_config = aea_conf.private_key_paths.read(FETCHAI) - if fetchai_private_key_config is None: - path = Path(FETCHAI_PRIVATE_KEY_FILE) - entity = Entity() - with open(path, "w+") as file: - file.write(entity.private_key_hex) - fetchai_private_key_path = FETCHAI_PRIVATE_KEY_FILE - fetchai_private_key_config = PrivateKeyPathConfig(FETCHAI, fetchai_private_key_path) - aea_conf.private_key_paths.create(fetchai_private_key_config.ledger, fetchai_private_key_config) - else: - fetchai_private_key_config = cast(PrivateKeyPathConfig, fetchai_private_key_config) - try: - _try_validate_fet_private_key_path(fetchai_private_key_config.path) - except FileNotFoundError: - logger.error("File {} for private key {} not found.".format(repr(fetchai_private_key_config.path), fetchai_private_key_config.ledger)) - sys.exit(-1) - - ethereum_private_key_config = aea_conf.private_key_paths.read(ETHEREUM) - if ethereum_private_key_config is None: - path = Path(ETHEREUM_PRIVATE_KEY_FILE) - account = Account.create() - with open(path, "w+") as file: - file.write(account.privateKey.hex()) - ethereum_private_key_path = ETHEREUM_PRIVATE_KEY_FILE - ethereum_private_key_config = PrivateKeyPathConfig(ETHEREUM, ethereum_private_key_path) - aea_conf.private_key_paths.create(ethereum_private_key_config.ledger, ethereum_private_key_config) - else: - ethereum_private_key_config = cast(PrivateKeyPathConfig, ethereum_private_key_config) - try: - _try_validate_ethereum_private_key_path(ethereum_private_key_config.path) - except FileNotFoundError: - logger.error("File {} for private key {} not found.".format(repr(ethereum_private_key_config.path), ethereum_private_key_config.ledger)) - sys.exit(-1) - - # update aea config - path = Path(DEFAULT_AEA_CONFIG_FILE) - fp = open(str(path), mode="w", encoding="utf-8") - agent_loader.dump(aea_conf, fp) - ctx.agent_config = aea_conf - - -def _verify_ledger_apis_access(ctx: Context) -> None: - """ - Verify access to ledger apis. - - :param ctx: Context - """ - path = Path(DEFAULT_AEA_CONFIG_FILE) - agent_loader = ConfigLoader("aea-config_schema.json", AgentConfig) - fp = open(str(path), mode="r", encoding="utf-8") - aea_conf = agent_loader.load(fp) - - for identifier, value in aea_conf.ledger_apis.read_all(): - if identifier not in SUPPORTED_LEDGER_APIS: - ValueError("Unsupported identifier in ledger apis.") - - fetchai_ledger_api_config = aea_conf.ledger_apis.read(FETCHAI) - if fetchai_ledger_api_config is None: - logger.debug("No fetchai ledger api config specified.") - else: - fetchai_ledger_api_config = cast(LedgerAPIConfig, fetchai_ledger_api_config) - try: - from fetchai.ledger.api import LedgerApi - LedgerApi(fetchai_ledger_api_config.addr, fetchai_ledger_api_config.port) - except Exception: - logger.error("Cannot connect to fetchai ledger with provided config.") - sys.exit(-1) - - ethereum_ledger_config = aea_conf.ledger_apis.read(ETHEREUM) - if ethereum_ledger_config is None: - logger.debug("No ethereum ledger api config specified.") - else: - ethereum_ledger_config = cast(LedgerAPIConfig, ethereum_ledger_config) - try: - from web3 import Web3, HTTPProvider - Web3(HTTPProvider(ethereum_ledger_config.addr)) - except Exception: - logger.error("Cannot connect to ethereum ledger with provided config.") - sys.exit(-1) - - -def _create_temporary_private_key() -> bytes: - """ - Create a temporary private key. - - :return: the private key in pem format. - """ - crypto = DefaultCrypto() - pem = crypto._private_key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) # type: ignore - return pem - - def _try_validate_private_key_pem_path(private_key_pem_path: str) -> None: """ Try to validate a private key. @@ -176,7 +46,7 @@ def _try_validate_private_key_pem_path(private_key_pem_path: str) -> None: DefaultCrypto(private_key_pem_path=private_key_pem_path) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_pem_path)) - sys.exit(-1) + sys.exit(1) def _try_validate_fet_private_key_path(private_key_path: str) -> None: @@ -193,7 +63,7 @@ def _try_validate_fet_private_key_path(private_key_path: str) -> None: Entity.from_hex(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - sys.exit(-1) + sys.exit(1) def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: @@ -210,17 +80,37 @@ def _try_validate_ethereum_private_key_path(private_key_path: str) -> None: Account.from_key(data) except ValueError: logger.error("This is not a valid private key file: '{}'".format(private_key_path)) - sys.exit(-1) + sys.exit(1) -def _create_temporary_private_key_pem_path() -> str: +def _create_default_private_key() -> None: """ - Create a temporary private key and path to the file. + Create a default private key. - :return: private_key_pem_path + :return: None + """ + crypto = DefaultCrypto() + with open(DEFAULT_PRIVATE_KEY_FILE, "wb") as file: + file.write(crypto.private_key_pem) + + +def _create_fetchai_private_key() -> None: + """ + Create a fetchai private key. + + :return: None + """ + entity = Entity() + with open(FETCHAI_PRIVATE_KEY_FILE, "w+") as file: + file.write(entity.private_key_hex) + + +def _create_ethereum_private_key() -> None: + """ + Create an ethereum private key. + + :return: None """ - pem = _create_temporary_private_key() - file = open(DEFAULT_PRIVATE_KEY_FILE, "wb") - file.write(pem) - file.close() - return DEFAULT_PRIVATE_KEY_FILE + account = Account.create() + with open(ETHEREUM_PRIVATE_KEY_FILE, "w+") as file: + file.write(account.privateKey.hex()) diff --git a/aea/crypto/ledger_apis.py b/aea/crypto/ledger_apis.py index c813281ae0..e3e10d94b7 100644 --- a/aea/crypto/ledger_apis.py +++ b/aea/crypto/ledger_apis.py @@ -21,6 +21,7 @@ """Module wrapping all the public and private keys cryptography.""" import logging +import sys import time from typing import Any, Dict, Optional, Tuple, cast @@ -35,6 +36,7 @@ DEFAULT_FETCHAI_CONFIG = ('alpha.fetch-ai.com', 80) SUCCESSFUL_TERMINAL_STATES = ('Executed', 'Submitted') +SUPPORTED_LEDGER_APIS = [ETHEREUM, FETCHAI] logger = logging.getLogger(__name__) @@ -202,3 +204,33 @@ def is_tx_settled(self, identifier: str, tx_digest: str, amount: int) -> bool: logger.warning("An error occured while attempting to check the transaction!") return is_successful + + +def _try_to_instantiate_fetchai_ledger_api(addr: str, port: int) -> None: + """ + Tro to instantiate the fetchai ledger api. + + :param addr: the address + :param port: the port + """ + try: + from fetchai.ledger.api import LedgerApi + LedgerApi(addr, port) + except Exception: + logger.error("Cannot connect to fetchai ledger with provided config.") + sys.exit(1) + + +def _try_to_instantiate_ethereum_ledger_api(addr: str, port: int) -> None: + """ + Tro to instantiate the fetchai ledger api. + + :param addr: the address + :param port: the port + """ + try: + from web3 import Web3, HTTPProvider + Web3(HTTPProvider(addr)) + except Exception: + logger.error("Cannot connect to ethereum ledger with provided config.") + sys.exit(1) diff --git a/aea/crypto/wallet.py b/aea/crypto/wallet.py index 8e41bf17ee..287642fad7 100644 --- a/aea/crypto/wallet.py +++ b/aea/crypto/wallet.py @@ -27,8 +27,6 @@ from aea.crypto.fetchai import FetchAICrypto, FETCHAI SUPPORTED_CRYPTOS = [DEFAULT, ETHEREUM, FETCHAI] -SUPPORTED_LEDGER_APIS = [ETHEREUM, FETCHAI] -CURRENCY_TO_ID_MAP = {'FET': FETCHAI, 'ETH': ETHEREUM} class Wallet(object): diff --git a/packages/skills/carpark_detection/handlers.py b/packages/skills/carpark_detection/handlers.py index 5012c18287..f1cbb93aa8 100644 --- a/packages/skills/carpark_detection/handlers.py +++ b/packages/skills/carpark_detection/handlers.py @@ -28,7 +28,7 @@ from aea.protocols.default.serialization import DefaultSerializer from aea.protocols.fipa.message import FIPAMessage from aea.protocols.fipa.serialization import FIPASerializer -from aea.protocols.oef.models import Query, Description +from aea.protocols.oef.models import Description, Query from aea.skills.base import Handler if TYPE_CHECKING: diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index 02d20d0dcd..ce6da23a50 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -40,7 +40,7 @@ def tests_private_keys(self): private_key_path = os.path.join(CUR_PATH, "data", "priv.pem") _try_validate_private_key_pem_path(private_key_path) with pytest.raises(SystemExit): - private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") + private_key_path = os.path.join(CUR_PATH, "data", "priv_wrong.pem") _try_validate_private_key_pem_path(private_key_path) private_key_path = os.path.join(CUR_PATH, "data", "fet_private_key.txt") From 6a9f2bd2da8aeefa81d9b56abb9ea0f646632d0e Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 14:48:17 +0000 Subject: [PATCH 19/38] Changes based on the PR's comments --- packages/skills/weather_client/behaviours.py | 6 +----- packages/skills/weather_client/handlers.py | 2 +- packages/skills/weather_station/behaviours.py | 2 +- packages/skills/weather_station/handlers.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/skills/weather_client/behaviours.py b/packages/skills/weather_client/behaviours.py index 818cdf0618..6729cab109 100644 --- a/packages/skills/weather_client/behaviours.py +++ b/packages/skills/weather_client/behaviours.py @@ -30,16 +30,12 @@ else: from weather_client_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_client_ledger_skill") +logger = logging.getLogger("aea.weather_client_skill") class MySearchBehaviour(Behaviour): """This class scaffolds a behaviour.""" - def __init__(self, **kwargs): - """Initialise the class.""" - super().__init__(**kwargs) - def setup(self) -> None: """Implement the setup for the behaviour.""" pass diff --git a/packages/skills/weather_client/handlers.py b/packages/skills/weather_client/handlers.py index 1d05b4171b..80fbbbe9ee 100644 --- a/packages/skills/weather_client/handlers.py +++ b/packages/skills/weather_client/handlers.py @@ -198,7 +198,7 @@ def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, d dialogue_id=dialogue_id, target=new_target_id, performative=FIPAMessage.Performative.INFORM, - json_data={"transaction": "Done"}) + json_data={"Done": "Sending payment via bank transfer"}) dialogue.outgoing_extend(inform_msg) self.context.outbox.put_message(to=counterparty_pbk, sender=self.context.agent_public_key, diff --git a/packages/skills/weather_station/behaviours.py b/packages/skills/weather_station/behaviours.py index 1dcbb5a21a..0686893b69 100644 --- a/packages/skills/weather_station/behaviours.py +++ b/packages/skills/weather_station/behaviours.py @@ -31,7 +31,7 @@ else: from weather_station_skill.strategy import Strategy -logger = logging.getLogger("aea.weather_station_ledger_skill") +logger = logging.getLogger("aea.weather_station_skill") SERVICE_ID = '' diff --git a/packages/skills/weather_station/handlers.py b/packages/skills/weather_station/handlers.py index 558aa96678..be87121fe0 100644 --- a/packages/skills/weather_station/handlers.py +++ b/packages/skills/weather_station/handlers.py @@ -236,7 +236,7 @@ def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogu sender[-5:])) json_data = cast(dict, msg.get("json_data")) - if "transaction" in json_data: + if "Done" in json_data: inform_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target, From 99d24efc400a4207a27e2194a50b90cd6c8f6384 Mon Sep 17 00:00:00 2001 From: Diarmid Campbell Date: Fri, 1 Nov 2019 14:56:06 +0000 Subject: [PATCH 20/38] Runs entire agent from aea run - fixed a load of bugs --- packages/skills/carpark_client/behaviours.py | 4 +- packages/skills/carpark_client/dialogues.py | 3 +- packages/skills/carpark_client/handlers.py | 16 +-- packages/skills/carpark_client/skill.yaml | 5 +- packages/skills/carpark_client/strategy.py | 15 ++- .../skills/carpark_detection/behaviours.py | 99 +++++++++++++++++-- .../carpark_detection/detection_database.py | 31 ++++-- packages/skills/carpark_detection/skill.yaml | 8 ++ packages/skills/carpark_detection/strategy.py | 27 ++++- 9 files changed, 164 insertions(+), 44 deletions(-) diff --git a/packages/skills/carpark_client/behaviours.py b/packages/skills/carpark_client/behaviours.py index e0ac898517..51e595b090 100644 --- a/packages/skills/carpark_client/behaviours.py +++ b/packages/skills/carpark_client/behaviours.py @@ -18,7 +18,6 @@ # ------------------------------------------------------------------------------ """This package contains a scaffold of a behaviour.""" -import datetime import logging from typing import cast, TYPE_CHECKING @@ -59,9 +58,8 @@ def act(self) -> None: """ strategy = cast(Strategy, self.context.strategy) if strategy.is_searching and strategy.is_time_to_search(): + strategy.on_submit_search() self._search_id += 1 - strategy.pause_search() - strategy.last_search_time = datetime.datetime.now() query = strategy.get_service_query() search_request = OEFMessage(oef_type=OEFMessage.Type.SEARCH_SERVICES, id=self._search_id, diff --git a/packages/skills/carpark_client/dialogues.py b/packages/skills/carpark_client/dialogues.py index f4d0e5d2a5..b8b566a43d 100644 --- a/packages/skills/carpark_client/dialogues.py +++ b/packages/skills/carpark_client/dialogues.py @@ -28,7 +28,6 @@ from enum import Enum import logging -import random from typing import Any, Dict, Optional, cast from aea.helpers.dialogue.base import DialogueLabel @@ -179,7 +178,7 @@ def _next_dialogue_id(self) -> int: :return: the next id """ - self._dialogue_id = random.randint(0, 1000000) + self._dialogue_id += 1 print("_next_dialogue_id: _dialogue_id = {}".format(self._dialogue_id)) return self._dialogue_id diff --git a/packages/skills/carpark_client/handlers.py b/packages/skills/carpark_client/handlers.py index a6c46718d1..6fd5586e8e 100644 --- a/packages/skills/carpark_client/handlers.py +++ b/packages/skills/carpark_client/handlers.py @@ -163,7 +163,6 @@ def _handle_propose(self, msg: FIPAMessage, sender: str, message_id: int, dialog else: logger.info("[{}]: declining the proposal from sender={}".format(self.context.agent_name, sender[-5:])) - strategy.unpause_search() decline_msg = FIPAMessage(message_id=new_message_id, dialogue_id=dialogue_id, target=new_target_id, @@ -185,15 +184,7 @@ def _handle_decline(self, msg: FIPAMessage, sender: str, message_id: int, dialog :param dialogue: the dialogue object :return: None """ - strategy = cast(Strategy, self.context.strategy) - strategy.unpause_search() logger.info("[{}]: received DECLINE from sender={}".format(self.context.agent_name, sender[-5:])) - # target = msg.get("target") - # dialogues = cast(Dialogues, self.context.dialogues) - # if target == 1: - # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.DECLINED_CFP) - # elif target == 3: - # dialogues.dialogue_stats.add_dialogue_endstate(Dialogue.EndState.DECLINED_ACCEPT) def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: """ @@ -294,6 +285,8 @@ def _handle_search(self, agents: List[str]) -> None: """ strategy = cast(Strategy, self.context.strategy) if len(agents) > 0: + strategy.on_search_success() + logger.info("[{}]: found agents={}, stopping search.".format(self.context.agent_name, list(map(lambda x: x[-5:], agents)))) # pick first agent found @@ -313,8 +306,8 @@ def _handle_search(self, agents: List[str]) -> None: protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(cfp_msg)) else: - strategy.unpause_search() logger.info("[{}]: found no agents, continue searching.".format(self.context.agent_name)) + strategy.on_search_failed() class MyTransactionHandler(Handler): @@ -338,8 +331,7 @@ def handle(self, message: Message, sender: str) -> None: if tx_msg_response is not None and \ TransactionMessage.Performative(tx_msg_response.get("performative")) == TransactionMessage.Performative.ACCEPT: logger.info("[{}]: transaction was successful.".format(self.context.agent_name)) - strategy = cast(Strategy, self.context.strategy) - strategy.unpause_search() + json_data = {'transaction_digest': tx_msg_response.get("transaction_digest")} dialogue_label = DialogueLabel.from_json(cast(Dict[str, str], tx_msg_response.get("dialogue_label"))) dialogues = cast(Dialogues, self.context.dialogues) diff --git a/packages/skills/carpark_client/skill.yaml b/packages/skills/carpark_client/skill.yaml index c840da00a3..0b9b67d50d 100644 --- a/packages/skills/carpark_client/skill.yaml +++ b/packages/skills/carpark_client/skill.yaml @@ -23,9 +23,10 @@ shared_classes: class_name: Strategy args: country: UK - search_interval: 10 + search_interval: 120 + no_find_search_interval: 5 max_price: 400000000 - max_detection_age: 3600000 + max_detection_age: 3600 - shared_class: class_name: Dialogues args: {} diff --git a/packages/skills/carpark_client/strategy.py b/packages/skills/carpark_client/strategy.py index d160c74eba..472c2223c0 100644 --- a/packages/skills/carpark_client/strategy.py +++ b/packages/skills/carpark_client/strategy.py @@ -31,6 +31,7 @@ DEFAULT_SEARCH_INTERVAL = 5.0 DEFAULT_MAX_PRICE = 4000 DEFAULT_MAX_DETECTION_AGE = 60 * 60 # 1 hour +DEFAULT_NO_FINDSEARCH_INTERVAL = 5 class Strategy(SharedClass): @@ -44,6 +45,7 @@ def __init__(self, **kwargs) -> None: """ self._country = kwargs.pop('country') if 'country' in kwargs.keys() else DEFAULT_COUNTRY self._search_interval = cast(float, kwargs.pop('search_interval')) if 'search_interval' in kwargs.keys() else DEFAULT_SEARCH_INTERVAL + self._no_find_search_interval = cast(float, kwargs.pop('no_find_search_interval')) if 'no_find_search_interval' in kwargs.keys() else DEFAULT_NO_FINDSEARCH_INTERVAL self._max_price = kwargs.pop('max_price') if 'max_price' in kwargs.keys() else DEFAULT_MAX_PRICE self._max_detection_age = kwargs.pop('max_detection_age') if 'max_detection_age' in kwargs.keys() else DEFAULT_MAX_DETECTION_AGE super().__init__(**kwargs) @@ -59,15 +61,20 @@ def get_service_query(self) -> Query: query = Query([Constraint('longitude', ConstraintType("!=", 0.0))], model=None) return query - def pause_search(self): - """Stop searching temporarily.""" + def on_submit_search(self): + """Call when you submit a search ( to suspend searching).""" self.is_searching = False - def unpause_search(self): - """Restart searching after pausing.""" + def on_search_success(self): + """Call when search returns succesfully.""" self.last_search_time = datetime.datetime.now() self.is_searching = True + def on_search_failed(self): + """Call when search returns with no matches.""" + self.last_search_time = datetime.datetime.now() - datetime.timedelta(seconds=self._search_interval - self._no_find_search_interval) + self.is_searching = True + def is_time_to_search(self) -> bool: """ Check whether it is time to search. diff --git a/packages/skills/carpark_detection/behaviours.py b/packages/skills/carpark_detection/behaviours.py index 7c01becf33..448954f08e 100644 --- a/packages/skills/carpark_detection/behaviours.py +++ b/packages/skills/carpark_detection/behaviours.py @@ -20,6 +20,8 @@ """This package contains a scaffold of a behaviour.""" import logging +import os +import subprocess from typing import cast, TYPE_CHECKING from aea.skills.base import Behaviour @@ -38,6 +40,81 @@ SERVICE_ID = '' +DEFAULT_LAT = 1 +DEFAULT_LON = 1 +DEFAULT_IMAGE_CAPTURE_INTERVAL = 300 + + +class CarParkDetectionAndGUIBehaviour(Behaviour): + """This class implements a behaviour.""" + + def __init__(self, **kwargs): + """Initialise the behaviour.""" + print("*****kwargs: {}".format(kwargs)) + self.image_capture_interval = kwargs.pop('image_capture_interval') if 'image_capture_interval' in kwargs.keys() else DEFAULT_IMAGE_CAPTURE_INTERVAL + self.default_latitude = kwargs.pop('default_latitude') if 'default_latitude' in kwargs.keys() else DEFAULT_LAT + self.default_longitude = kwargs.pop('default_longitude') if 'default_longitude' in kwargs.keys() else DEFAULT_LON + super().__init__(**kwargs) + + def setup(self) -> None: + """ + Implement the setup. + + :return: None + """ + logger.info("[{}]: Attempt to launch car park detection and GUI in seperate processes.".format(self.context.agent_name)) + old_cwp = os.getcwd() + os.chdir('../') + strategy = cast(Strategy, self.context.strategy) + if os.path.isfile('run_scripts/run_carparkagent.py'): + param_list = [ + 'python', 'run_scripts/run_carparkagent.py', + '-ps', str(self.image_capture_interval), + '-lat', str(self.default_latitude), + '-lon', str(self.default_longitude)] + logger.info("[{}]:Launchng process {}".format(self.context.agent_name, param_list)) + self.process_id = subprocess.Popen(param_list) + os.chdir(old_cwp) + logger.info("[{}]: detection and gui process launched, process_id {}".format(self.context.agent_name, self.process_id)) + strategy.other_carpark_processes_running = True + else: + logger.info("[{}]: Failed to find run_carpakragent.py - either you are running this without the rest of the carpark agent code (which can be got from here: https://github.com/fetchai/carpark_agent or you are running the aea from the wrong directory.".format(self.context.agent_name)) + + def act(self) -> None: + """ + Implement the act. + + :return: None + """ + """Return the state of the execution.""" + + # We never started the other processes + if self.process_id is None: + return + + return_code = self.process_id.poll() + + # Other procssess running fine + if return_code is None: + return + # Other processes have finished so we should finish too + # this is a bit hacky! + else: + exit() + + def teardown(self) -> None: + """ + Implement the task teardown. + + :return: None + """ + if self.process_id is None: + return + + self.process_id.terminate() + self.process_id.wait() + + class ServiceRegistrationBehaviour(Behaviour): """This class implements a behaviour.""" @@ -55,8 +132,18 @@ def setup(self) -> None: balance = self.context.ledger_apis.token_balance('fetchai', cast(str, self.context.agent_addresses.get('fetchai'))) logger.info("[{}]: starting balance on fetchai ledger={}.".format(self.context.agent_name, balance)) - if not self._registered: - strategy = cast(Strategy, self.context.strategy) + + def act(self) -> None: + """ + Implement the act. + + :return: None + """ + if self._registered: + return + + strategy = cast(Strategy, self.context.strategy) + if strategy.has_service_description(): desc = strategy.get_service_description() msg = OEFMessage(oef_type=OEFMessage.Type.REGISTER_SERVICE, id=REGISTER_ID, @@ -69,14 +156,6 @@ def setup(self) -> None: logger.info("[{}]: registering car park detection services on OEF.".format(self.context.agent_name)) self._registered = True - def act(self) -> None: - """ - Implement the act. - - :return: None - """ - pass - def teardown(self) -> None: """ Implement the task teardown. diff --git a/packages/skills/carpark_detection/detection_database.py b/packages/skills/carpark_detection/detection_database.py index 102d7c7a7b..ad4d06dbf4 100644 --- a/packages/skills/carpark_detection/detection_database.py +++ b/packages/skills/carpark_detection/detection_database.py @@ -41,11 +41,19 @@ def __init__(self, temp_dir, create_if_not_present=True): self.default_mask_ref_path = self.this_dir + "/default_mask_ref.png" self.num_digits_time = 12 # need to match this up with the generate functions below self.image_file_ext = ".png" - self.database_path = self.temp_dir + "/" + "detection_results.db" + if create_if_not_present: self.initialise_backend() + def is_db_exits(self): + """Return true if database exixts and is set up.""" + if not os.path.isfile(self.database_path): + return False + + ret = self.get_system_status("db", False) == "Exists" + return ret + def reset_database(self): """Reset the database and remove all data.""" # If we need to reset the database, then remove the table and any stored images @@ -60,7 +68,9 @@ def reset_database(self): shutil.rmtree(self.processed_image_dir) # Recreate them + print("initialise_backend") self.initialise_backend() + print("FINISH initialise_backend") def reset_mask(self): """Just reset the detection mask.""" @@ -75,13 +85,14 @@ def reset_mask(self): self.ensure_dirs_exist() def initialise_backend(self): - """Generate all tables in database and any temporary directories needed.""" + """Set up database and initialise the tables.""" self.ensure_dirs_exist() self.execute_single_sql( "CREATE TABLE IF NOT EXISTS images (epoch INTEGER, raw_image_path TEXT, " "processed_image_path TEXT, total_count INTEGER, " "moving_count INTEGER, free_spaces INTEGER, lat TEXT, lon TEXT)") + # self.execute_single_sql("DROP TABLE fet_table") self.execute_single_sql( "CREATE TABLE IF NOT EXISTS fet_table (id INTEGER PRIMARY KEY, amount BIGINT, last_updated TEXT)") @@ -99,6 +110,13 @@ def initialise_backend(self): self.execute_single_sql( "CREATE TABLE IF NOT EXISTS dialogue_statuses (dialogue_id TEXT, epoch DECIMAL, other_agent_key TEXT, received_msg TEXT, sent_msg TEXT)") + if not self.is_db_exits(): + self.set_system_status("lat", "UNKNOWN") + self.set_system_status("lon", "UNKNOWN") + self.set_system_status("db", "Exists") + + print("**** backend initialised") + def set_fet(self, amount, t): """Record how much FET we have and when we last read it from the ledger.""" self.execute_single_sql( @@ -144,9 +162,9 @@ def set_system_status(self, system_name, status): self.execute_single_sql( "INSERT OR REPLACE INTO status_table(system_name, status) values('{}', '{}')".format(system_name, status)) - def get_system_status(self, system_name): + def get_system_status(self, system_name, print_exceptions=True): """Read the status of one of the systems.""" - result = self.execute_single_sql("SELECT status FROM status_table WHERE system_name='{}'".format(system_name)) + result = self.execute_single_sql("SELECT status FROM status_table WHERE system_name='{}'".format(system_name), print_exceptions) if len(result) != 0: return result[0][0] else: @@ -287,7 +305,7 @@ def add_entry(self, raw_image, processed_image, total_count, moving_count, free_ self.execute_single_sql("INSERT INTO images VALUES ({}, '{}', '{}', {}, {}, {}, '{}', '{}')".format( t, raw_path, processed_path, total_count, moving_count, free_spaces, lat, lon)) - def execute_single_sql(self, command): + def execute_single_sql(self, command, print_exceptions=True): """Query the database - all the other functions use this under the hood.""" conn = None ret = [] @@ -298,7 +316,8 @@ def execute_single_sql(self, command): ret = c.fetchall() conn.commit() except Exception as e: - print("Exception in database: {}".format(e)) + if print_exceptions: + print("Exception in database: {}".format(e)) finally: if conn is not None: conn.close() diff --git a/packages/skills/carpark_detection/skill.yaml b/packages/skills/carpark_detection/skill.yaml index 548d1e1658..b5c0d46825 100644 --- a/packages/skills/carpark_detection/skill.yaml +++ b/packages/skills/carpark_detection/skill.yaml @@ -7,6 +7,13 @@ behaviours: - behaviour: class_name: ServiceRegistrationBehaviour args: {} + - behaviour: + class_name: CarParkDetectionAndGUIBehaviour + args: + default_longitude: -73.967491 + default_latitude: 40.780343 + image_capture_interval: 120 + handlers: - handler: class_name: FIPAHandler @@ -19,6 +26,7 @@ shared_classes: data_price_fet: 200000000 db_is_rel_to_cwd: true db_rel_dir: ../temp_files + - shared_class: class_name: Dialogues args: {} diff --git a/packages/skills/carpark_detection/strategy.py b/packages/skills/carpark_detection/strategy.py index 3ec1b1cda7..717d6d36e6 100644 --- a/packages/skills/carpark_detection/strategy.py +++ b/packages/skills/carpark_detection/strategy.py @@ -67,24 +67,37 @@ def __init__(self, **kwargs) -> None: print("WARNING - DATABASE dir does not exist") self.db = DetectionDatabase(db_dir, False) - self.lat = 43 - self.lon = 42 self.record_balance(balance) + self.other_carpark_processes_running = False def record_balance(self, balance): """Record current balance to database.""" self.db.set_fet(balance, time.time()) + def has_service_description(self): + """Return true if we have a description""" + if not self.db.is_db_exits(): + return False + + lat, lon = self.db.get_lat_lon() + if lat is None or lon is None: + return False + + return True + def get_service_description(self) -> Description: """ Get the service description. :return: a description of the offered services """ + assert(self.has_service_description()) + + lat, lon = self.db.get_lat_lon() desc = Description( { - "latitude": float(self.lat), - "longitude": float(self.lon), + "latitude": lat, + "longitude": lon, "unique_id": self.context.agent_public_key }, data_model=CarParkDataModel() ) @@ -103,6 +116,9 @@ def is_matching_supply(self, query: Query) -> bool: def has_data(self) -> bool: """Return whether we have any useful data to sell.""" + if not self.db.is_db_exits(): + return False + data = self.db.get_latest_detection_data(1) return len(data) > 0 @@ -113,7 +129,8 @@ def generate_proposal_and_data(self, query: Query) -> Tuple[Description, Dict[st :param query: the query :return: a tuple of proposal and the bytes of carpark data """ - # TODO, this is a stub + assert(self.db.is_db_exits()) + data = self.db.get_latest_detection_data(1) assert (len(data) > 0) From 2a1f1208481c5a27839844b0e409841a145586a3 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Nov 2019 15:16:35 +0000 Subject: [PATCH 21/38] Prepare develop for release v0.1.12 --- HISTORY.rst | 8 +++++++- aea/__version__.py | 2 +- deploy-image/docker-env.sh | 2 +- develop-image/docker-env.sh | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 559f458c4f..f989a05514 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,7 +11,6 @@ Release History - Provides examples and fixes. - 0.1.2 (2019-09-16) ------------------- @@ -92,3 +91,10 @@ Release History - Adds ledger integrations for fetch.ai and ethereum - Adds carpark examples and ledger examples - Multiple additional minor fixes and changes + +0.1.12 (2019-11-01) +------------------- + +- Fixes some examples and docs +- Refactors crypto modules and adds additional tests +- Multiple additional minor fixes and changes diff --git a/aea/__version__.py b/aea/__version__.py index 40cd04786a..d081cf74d9 100644 --- a/aea/__version__.py +++ b/aea/__version__.py @@ -23,7 +23,7 @@ __title__ = 'aea' __description__ = 'Autonomous Economic Agent framework' __url__ = 'https://github.com/fetchai/agents-aea.git' -__version__ = '0.1.11' +__version__ = '0.1.12' __author__ = 'Fetch.AI Limited' __license__ = 'Apache 2.0' __copyright__ = '2019 Fetch.AI Limited' diff --git a/deploy-image/docker-env.sh b/deploy-image/docker-env.sh index 19ead4f5cf..06ca5fa060 100755 --- a/deploy-image/docker-env.sh +++ b/deploy-image/docker-env.sh @@ -1,7 +1,7 @@ #!/bin/bash # Swap the following lines if you want to work with 'latest' -DOCKER_IMAGE_TAG=aea-deploy:0.1.11 +DOCKER_IMAGE_TAG=aea-deploy:0.1.12 # DOCKER_IMAGE_TAG=aea-deploy:latest DOCKER_BUILD_CONTEXT_DIR=.. diff --git a/develop-image/docker-env.sh b/develop-image/docker-env.sh index c601e5b64a..5a6f5e5bd4 100755 --- a/develop-image/docker-env.sh +++ b/develop-image/docker-env.sh @@ -1,7 +1,7 @@ #!/bin/bash # Swap the following lines if you want to work with 'latest' -DOCKER_IMAGE_TAG=aea-develop:0.1.11 +DOCKER_IMAGE_TAG=aea-develop:0.1.12 # DOCKER_IMAGE_TAG=aea-develop:latest DOCKER_BUILD_CONTEXT_DIR=.. From 4d8351653f98bf3d4e52df879ce9be80e52e6827 Mon Sep 17 00:00:00 2001 From: Aristotelis Date: Fri, 1 Nov 2019 15:29:31 +0000 Subject: [PATCH 22/38] Commented out the external APIs call --- tests/test_crypto/test_ledger_apis.py | 87 +++++++++++++++++---------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/tests/test_crypto/test_ledger_apis.py b/tests/test_crypto/test_ledger_apis.py index 58dff81082..1bed1abfb0 100644 --- a/tests/test_crypto/test_ledger_apis.py +++ b/tests/test_crypto/test_ledger_apis.py @@ -20,13 +20,15 @@ """This module contains the tests for the crypto/helpers module.""" import logging -import os +from unittest import mock + import pytest -from aea.crypto.ethereum import ETHEREUM, EthereumCrypto -from aea.crypto.fetchai import FETCHAI, FetchAICrypto -from aea.crypto.ledger_apis import LedgerApis, DEFAULT_FETCHAI_CONFIG -from tests.conftest import CUR_PATH +from aea.crypto.ethereum import ETHEREUM +from aea.crypto.fetchai import FETCHAI +from aea.crypto.ledger_apis import LedgerApis, DEFAULT_FETCHAI_CONFIG, _try_to_instantiate_fetchai_ledger_api, \ + _try_to_instantiate_ethereum_ledger_api + logger = logging.getLogger(__name__) @@ -40,9 +42,11 @@ class TestLedgerApis: def test_initialisation(self): """Test the initialisation of the ledger APIs.""" - ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG}) + ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, + FETCHAI: DEFAULT_FETCHAI_CONFIG}) assert ledger_apis.configs.get(ETHEREUM) == DEFAULT_ETHEREUM_CONFIG - + assert ledger_apis.has_fetchai + assert ledger_apis.has_ethereum unknown_config = ("UknownPath", 8080) with pytest.raises(ValueError): ledger_apis = LedgerApis({"UNKNOWN": unknown_config}) @@ -52,34 +56,51 @@ def test_token_balance(self): ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, FETCHAI: DEFAULT_FETCHAI_CONFIG}) - balance = ledger_apis.token_balance(FETCHAI, eth_address) - assert balance == 0 - balance = ledger_apis.token_balance(ETHEREUM, eth_address) - assert balance != 0, "The specific address has some eth" - balance = ledger_apis.token_balance(ETHEREUM, fet_address) - assert balance == 0, "Should trigger the Exception and the balance will be 0" + with mock.patch.object(ledger_apis, 'token_balance', return_value=10): + balance = ledger_apis.token_balance(FETCHAI, eth_address) + assert balance == 10 + balance = ledger_apis.token_balance(ETHEREUM, eth_address) + assert balance == 10, "The specific address has some eth" + with mock.patch.object(ledger_apis, 'token_balance', return_value=0): + balance = ledger_apis.token_balance(ETHEREUM, fet_address) + assert balance == 0, "Should trigger the Exception and the balance will be 0" + # with mock.patch.object(ledger_apis, 'token_balance', return_value=Exception): + # balance = ledger_apis.token_balance(ETHEREUM, fet_address) + # assert balance == 0, "Should trigger the Exception and the balance will be 0" with pytest.raises(AssertionError): balance = ledger_apis.token_balance("UNKNOWN", fet_address) assert balance == 0, "Unknown identifier so it will return 0" - def test_transfer(self): - """Test the transfer function for the supported tokens.""" - private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") - eth_obj = EthereumCrypto(private_key_path=private_key_path) - private_key_path = os.path.join(CUR_PATH, 'data', "fet_private_key.txt") - fet_obj = FetchAICrypto(private_key_path=private_key_path) - ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, - FETCHAI: DEFAULT_FETCHAI_CONFIG}) - tx_digest = ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) - assert tx_digest is not None - assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) - tx_digest = "unknown_hash" - assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) + # def test_transfer(self): + # """Test the transfer function for the supported tokens.""" + # private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") + # eth_obj = EthereumCrypto(private_key_path=private_key_path) + # private_key_path = os.path.join(CUR_PATH, 'data', "fet_private_key.txt") + # fet_obj = FetchAICrypto(private_key_path=private_key_path) + # ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, + # FETCHAI: DEFAULT_FETCHAI_CONFIG}) + # + # with mock.patch.object(ledger_apis, 'transfer', + # return_value= "97fcacaaf94b62318c4e4bbf53fd2608c15062f17a6d1bffee0ba7af9b710e35"): + # tx_digest = ledger_apis.transfer(FETCHAI, fet_obj, fet_address, amount=10, tx_fee=10) + # assert tx_digest is not None + # with mock.patch.object(ledger_apis, 'is_tx_settled', return_value= True): + # assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) + # with mock.patch.object(ledger_apis, 'is_tx_settled', return_value= False): + # assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) + # with mock.patch.object(ledger_apis, 'transfer', + # return_value="97fcacaaf94b62318c4e4bbf53fd2608c15062f17a6d1bffee0ba7af9b710e35"): + # tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=10, tx_fee=200000) + # assert tx_digest is not None + # with mock.patch.object(ledger_apis, 'is_tx_settled', return_value= True): + # assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) + # with mock.patch.object(ledger_apis, 'is_tx_settled', return_value= False): + # assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) - tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=10, tx_fee=200000) - assert tx_digest is not None - assert ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest, amount=10) - tx_digest = "unknown_hash" - assert not ledger_apis.is_tx_settled(identifier=ETHEREUM, tx_digest=tx_digest, amount=10) - with pytest.raises(ValueError): - tx_digest = ledger_apis.transfer(ETHEREUM, eth_obj, eth_address, amount=50, tx_fee=2) + def test_try_to_instantiate_fetchai_ledger_api(self): + """Test the instantiation of the fetchai ledger api.""" + _try_to_instantiate_fetchai_ledger_api(addr="127.0.0.1", port=80) + + def test__try_to_instantiate_ethereum_ledger_api(self): + """Test the instantiation of the ethereum ledger api.""" + _try_to_instantiate_ethereum_ledger_api(addr="127.0.0.1", port=80) From 934bd3425edc595d87e110c895217ae7c2dc3aa3 Mon Sep 17 00:00:00 2001 From: Diarmid Campbell Date: Fri, 1 Nov 2019 16:22:38 +0000 Subject: [PATCH 23/38] minor fixes --- packages/skills/carpark_detection/behaviours.py | 1 + packages/skills/carpark_detection/strategy.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/skills/carpark_detection/behaviours.py b/packages/skills/carpark_detection/behaviours.py index 448954f08e..ca1a7e0dd5 100644 --- a/packages/skills/carpark_detection/behaviours.py +++ b/packages/skills/carpark_detection/behaviours.py @@ -54,6 +54,7 @@ def __init__(self, **kwargs): self.image_capture_interval = kwargs.pop('image_capture_interval') if 'image_capture_interval' in kwargs.keys() else DEFAULT_IMAGE_CAPTURE_INTERVAL self.default_latitude = kwargs.pop('default_latitude') if 'default_latitude' in kwargs.keys() else DEFAULT_LAT self.default_longitude = kwargs.pop('default_longitude') if 'default_longitude' in kwargs.keys() else DEFAULT_LON + self.process_id = None super().__init__(**kwargs) def setup(self) -> None: diff --git a/packages/skills/carpark_detection/strategy.py b/packages/skills/carpark_detection/strategy.py index 717d6d36e6..b46375167c 100644 --- a/packages/skills/carpark_detection/strategy.py +++ b/packages/skills/carpark_detection/strategy.py @@ -75,7 +75,7 @@ def record_balance(self, balance): self.db.set_fet(balance, time.time()) def has_service_description(self): - """Return true if we have a description""" + """Return true if we have a description.""" if not self.db.is_db_exits(): return False From 89d11aca2dd9590142730dac8909ae7c07c1c741 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Nov 2019 16:55:51 +0000 Subject: [PATCH 24/38] Remove excessive logging --- packages/skills/weather_client/handlers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/skills/weather_client/handlers.py b/packages/skills/weather_client/handlers.py index 80fbbbe9ee..62be90ce71 100644 --- a/packages/skills/weather_client/handlers.py +++ b/packages/skills/weather_client/handlers.py @@ -71,7 +71,6 @@ def handle(self, message: Message, sender: str) -> None: # recover dialogue dialogues = cast(Dialogues, self.context.dialogues) - logger.info(fipa_msg) if dialogues.is_belonging_to_registered_dialogue(fipa_msg, sender, self.context.agent_public_key): dialogue = dialogues.get_dialogue(dialogue_id, sender, self.context.agent_public_key) dialogue.incoming_extend(fipa_msg) @@ -87,7 +86,6 @@ def handle(self, message: Message, sender: str) -> None: elif msg_performative == FIPAMessage.Performative.MATCH_ACCEPT_W_ADDRESS: self._handle_match_accept(fipa_msg, sender, message_id, dialogue_id, dialogue) elif msg_performative == FIPAMessage.Performative.INFORM: - logger.info("RECEIVED AN INFORM MESSAGE!!!!") self._handle_inform(fipa_msg, sender, message_id, dialogue_id, dialogue) def teardown(self) -> None: @@ -204,8 +202,8 @@ def _handle_match_accept(self, msg: FIPAMessage, sender: str, message_id: int, d sender=self.context.agent_public_key, protocol_id=FIPAMessage.protocol_id, message=FIPASerializer().encode(inform_msg)) - logger.info("[{}]: informing counterparty={} of transaction digest.".format(self.context.agent_name, - counterparty_pbk[-5:])) + logger.info("[{}]: informing counterparty={} of payment.".format(self.context.agent_name, + counterparty_pbk[-5:])) self._received_tx_message = True def _handle_inform(self, msg: FIPAMessage, sender: str, message_id: int, dialogue_id: int, dialogue: Dialogue) -> None: From 728245ba71096f016599868f0df19d4c00f4db83 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Nov 2019 17:39:33 +0000 Subject: [PATCH 25/38] Comment out ledger api tests --- tests/test_crypto/test_ledger_apis.py | 55 +++++++++++++-------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/tests/test_crypto/test_ledger_apis.py b/tests/test_crypto/test_ledger_apis.py index 1bed1abfb0..f7bd8d676e 100644 --- a/tests/test_crypto/test_ledger_apis.py +++ b/tests/test_crypto/test_ledger_apis.py @@ -20,14 +20,13 @@ """This module contains the tests for the crypto/helpers module.""" import logging -from unittest import mock +# from unittest import mock import pytest from aea.crypto.ethereum import ETHEREUM from aea.crypto.fetchai import FETCHAI -from aea.crypto.ledger_apis import LedgerApis, DEFAULT_FETCHAI_CONFIG, _try_to_instantiate_fetchai_ledger_api, \ - _try_to_instantiate_ethereum_ledger_api +from aea.crypto.ledger_apis import LedgerApis, DEFAULT_FETCHAI_CONFIG # , _try_to_instantiate_fetchai_ledger_api, _try_to_instantiate_ethereum_ledger_api logger = logging.getLogger(__name__) @@ -51,26 +50,25 @@ def test_initialisation(self): with pytest.raises(ValueError): ledger_apis = LedgerApis({"UNKNOWN": unknown_config}) - def test_token_balance(self): - """Test the token_balance for the different tokens.""" - ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, - FETCHAI: DEFAULT_FETCHAI_CONFIG}) - - with mock.patch.object(ledger_apis, 'token_balance', return_value=10): - balance = ledger_apis.token_balance(FETCHAI, eth_address) - assert balance == 10 - balance = ledger_apis.token_balance(ETHEREUM, eth_address) - assert balance == 10, "The specific address has some eth" - with mock.patch.object(ledger_apis, 'token_balance', return_value=0): - balance = ledger_apis.token_balance(ETHEREUM, fet_address) - assert balance == 0, "Should trigger the Exception and the balance will be 0" - # with mock.patch.object(ledger_apis, 'token_balance', return_value=Exception): - # balance = ledger_apis.token_balance(ETHEREUM, fet_address) - # assert balance == 0, "Should trigger the Exception and the balance will be 0" - with pytest.raises(AssertionError): - balance = ledger_apis.token_balance("UNKNOWN", fet_address) - assert balance == 0, "Unknown identifier so it will return 0" + # def test_token_balance(self): + # """Test the token_balance for the different tokens.""" + # ledger_apis = LedgerApis({ETHEREUM: DEFAULT_ETHEREUM_CONFIG, + # FETCHAI: DEFAULT_FETCHAI_CONFIG}) + # with mock.patch.object(ledger_apis, 'token_balance', return_value=10): + # balance = ledger_apis.token_balance(FETCHAI, eth_address) + # assert balance == 10 + # balance = ledger_apis.token_balance(ETHEREUM, eth_address) + # assert balance == 10, "The specific address has some eth" + # with mock.patch.object(ledger_apis, 'token_balance', return_value=0): + # balance = ledger_apis.token_balance(ETHEREUM, fet_address) + # assert balance == 0, "Should trigger the Exception and the balance will be 0" + # with mock.patch.object(ledger_apis, 'token_balance', return_value=Exception): + # balance = ledger_apis.token_balance(ETHEREUM, fet_address) + # assert balance == 0, "Should trigger the Exception and the balance will be 0" + # with pytest.raises(AssertionError): + # balance = ledger_apis.token_balance("UNKNOWN", fet_address) + # assert balance == 0, "Unknown identifier so it will return 0" # def test_transfer(self): # """Test the transfer function for the supported tokens.""" # private_key_path = os.path.join(CUR_PATH, "data", "eth_private_key.txt") @@ -96,11 +94,10 @@ def test_token_balance(self): # assert ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) # with mock.patch.object(ledger_apis, 'is_tx_settled', return_value= False): # assert not ledger_apis.is_tx_settled(identifier=FETCHAI, tx_digest=tx_digest, amount=10) + # def test_try_to_instantiate_fetchai_ledger_api(self): + # """Test the instantiation of the fetchai ledger api.""" + # _try_to_instantiate_fetchai_ledger_api(addr="127.0.0.1", port=80) - def test_try_to_instantiate_fetchai_ledger_api(self): - """Test the instantiation of the fetchai ledger api.""" - _try_to_instantiate_fetchai_ledger_api(addr="127.0.0.1", port=80) - - def test__try_to_instantiate_ethereum_ledger_api(self): - """Test the instantiation of the ethereum ledger api.""" - _try_to_instantiate_ethereum_ledger_api(addr="127.0.0.1", port=80) + # def test__try_to_instantiate_ethereum_ledger_api(self): + # """Test the instantiation of the ethereum ledger api.""" + # _try_to_instantiate_ethereum_ledger_api(addr="127.0.0.1", port=80) From 94f9caddbb52b639a61212b41730ef556bd56300 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 1 Nov 2019 19:16:55 +0100 Subject: [PATCH 26/38] remove pending tests. --- aea/connections/p2p/base.py | 4 +- tests/test_connections/test_p2p.py | 94 +++++++++++++----------------- 2 files changed, 44 insertions(+), 54 deletions(-) diff --git a/aea/connections/p2p/base.py b/aea/connections/p2p/base.py index f1d123dcad..81ea5bdb2b 100644 --- a/aea/connections/p2p/base.py +++ b/aea/connections/p2p/base.py @@ -64,7 +64,7 @@ async def _send(self, writer: StreamWriter, data: bytes) -> None: """Send bytes.""" logger.debug("Send a message") nbytes = struct.pack("I", len(data)) - logger.debug("#bytes: {}".format(nbytes)) + logger.debug("#bytes: {!r}".format(nbytes)) try: writer.write(nbytes) writer.write(data) @@ -82,7 +82,7 @@ async def _recv_loop(self, reader) -> None: data = await self._recv(reader) if data is None: return - logger.debug("Message received: {}".format(data)) + logger.debug("Message received: {!r}".format(data)) envelope = Envelope.decode(data) # TODO handle decoding error logger.debug("Decoded envelope: {}".format(envelope)) self.in_queue.put_nowait(envelope) diff --git a/tests/test_connections/test_p2p.py b/tests/test_connections/test_p2p.py index 33437e472b..76bcf7669c 100644 --- a/tests/test_connections/test_p2p.py +++ b/tests/test_connections/test_p2p.py @@ -18,56 +18,46 @@ # ------------------------------------------------------------------------------ """This module contains the tests of the P2P channel.""" -import logging -import shutil -from pathlib import Path -from aea.connections.p2p.tcp_client import TCPClientChannel, TCPClientConnection -from aea.connections.p2p.tcp_server import TCPServerConnection, TCPServerChannel -from aea.mail.base import MailBox, Envelope -from aea.protocols.default.message import DefaultMessage -from aea.protocols.default.serialization import DefaultSerializer - - -class TestTCP: - """Test TCP connections.""" - - @classmethod - def setup_class(cls): - """Set up the test class.""" - p = Path("/tmp/aea/test_tcp/") - shutil.rmtree(str(p), ignore_errors=True) - p.mkdir(parents=True) - - socket_path = str(Path(p, "test_socket")) - cls.server_pbk, cls.client_pbk = "server_pbk", "client_pbk" - server_conn = TCPServerConnection(cls.server_pbk, TCPServerChannel(cls.server_pbk, socket_path, unix=True)) - client_conn = TCPClientConnection(cls.client_pbk, TCPClientChannel(cls.client_pbk, socket_path, unix=True)) - - cls.server_mailbox = MailBox(server_conn) - cls.client_mailbox = MailBox(client_conn) - - cls.server_mailbox.connect() - cls.client_mailbox.connect() - - def test_communication(self): - """Test that we are able to send an envelope from client to server.""" - msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello server") - expected_envelope = Envelope(to=self.server_pbk, sender=self.client_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) - self.client_mailbox.outbox.put(expected_envelope) - actual_envelope = self.server_mailbox.inbox.get(timeout=2.0) - logging.debug(actual_envelope) - assert expected_envelope == actual_envelope - - msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello client") - expected_envelope = Envelope(to=self.client_pbk, sender=self.server_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) - self.server_mailbox.outbox.put(expected_envelope) - actual_envelope = self.client_mailbox.inbox.get(timeout=30.0) - logging.debug(actual_envelope) - assert expected_envelope == actual_envelope - - @classmethod - def teardown_class(cls): - """Tear down the test class.""" - cls.server_mailbox.disconnect() - cls.client_mailbox.disconnect() +# class TestTCP: +# """Test TCP connections.""" +# +# @classmethod +# def setup_class(cls): +# """Set up the test class.""" +# p = Path("/tmp/aea/test_tcp/") +# shutil.rmtree(str(p), ignore_errors=True) +# p.mkdir(parents=True) +# +# socket_path = str(Path(p, "test_socket")) +# cls.server_pbk, cls.client_pbk = "server_pbk", "client_pbk" +# server_conn = TCPServerConnection(cls.server_pbk, TCPServerChannel(cls.server_pbk, socket_path, unix=True)) +# client_conn = TCPClientConnection(cls.client_pbk, TCPClientChannel(cls.client_pbk, socket_path, unix=True)) +# +# cls.server_mailbox = MailBox(server_conn) +# cls.client_mailbox = MailBox(client_conn) +# +# cls.server_mailbox.connect() +# cls.client_mailbox.connect() +# +# def test_communication(self): +# """Test that we are able to send an envelope from client to server.""" +# msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello server") +# expected_envelope = Envelope(to=self.server_pbk, sender=self.client_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) +# self.client_mailbox.outbox.put(expected_envelope) +# actual_envelope = self.server_mailbox.inbox.get(timeout=2.0) +# logging.debug(actual_envelope) +# assert expected_envelope == actual_envelope +# +# msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello client") +# expected_envelope = Envelope(to=self.client_pbk, sender=self.server_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) +# self.server_mailbox.outbox.put(expected_envelope) +# actual_envelope = self.client_mailbox.inbox.get(timeout=30.0) +# logging.debug(actual_envelope) +# assert expected_envelope == actual_envelope +# +# @classmethod +# def teardown_class(cls): +# """Tear down the test class.""" +# cls.server_mailbox.disconnect() +# cls.client_mailbox.disconnect() From 68fba68f938f1f479a558a883fe6da3ed3bf6082 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 1 Nov 2019 19:49:51 +0100 Subject: [PATCH 27/38] add envelope serializer. --- aea/mail/base.py | 72 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/aea/mail/base.py b/aea/mail/base.py index bef8da6c6d..34187ab0a0 100644 --- a/aea/mail/base.py +++ b/aea/mail/base.py @@ -20,6 +20,7 @@ """Mail module abstract base classes.""" import logging +from abc import ABC, abstractmethod from queue import Queue from typing import Optional, TYPE_CHECKING @@ -31,13 +32,55 @@ logger = logging.getLogger(__name__) +class EnvelopeSerializer(ABC): + """This abstract class let the devloper to specify serialization layer for the envelope.""" + + @abstractmethod + def encode(self, envelope: 'Envelope') -> bytes: + """Encode the envelope""" + + @abstractmethod + def decode(self, envelope_bytes: bytes) -> 'Envelope': + """Decode the envelope""" + + +class DefaultEnvelopeSerializer(EnvelopeSerializer): + """Default envelope serializer.""" + + def encode(self, envelope: 'Envelope') -> bytes: + """Encode the envelope""" + envelope_pb = base_pb2.Envelope() + envelope_pb.to = envelope.to + envelope_pb.sender = envelope.sender + envelope_pb.protocol_id = envelope.protocol_id + envelope_pb.message = envelope.message + + envelope_bytes = envelope_pb.SerializeToString() + return envelope_bytes + + def decode(self, envelope_bytes: bytes) -> 'Envelope': + """Decode the envelope""" + envelope_pb = base_pb2.Envelope() + envelope_pb.ParseFromString(envelope_bytes) + + to = envelope_pb.to + sender = envelope_pb.sender + protocol_id = envelope_pb.protocol_id + message = envelope_pb.message + + envelope = Envelope(to=to, sender=sender, + protocol_id=protocol_id, message=message) + return envelope + + class Envelope: """The top level message class.""" def __init__(self, to: Address, sender: Address, protocol_id: ProtocolId, - message: bytes): + message: bytes, + serializer: Optional[EnvelopeSerializer] = None): """ Initialize a Message object. @@ -45,11 +88,13 @@ def __init__(self, to: Address, :param sender: the public key of the sender. :param protocol_id: the protocol id. :param message: the protocol-specific message + :param serializer: the implementation for the envelope serialization. """ self._to = to self._sender = sender self._protocol_id = protocol_id self._message = message + self._serializer = serializer @property def to(self) -> Address: @@ -105,34 +150,21 @@ def encode(self) -> bytes: :return: the encoded envelope. """ - envelope = self - envelope_pb = base_pb2.Envelope() - envelope_pb.to = envelope.to - envelope_pb.sender = envelope.sender - envelope_pb.protocol_id = envelope.protocol_id - envelope_pb.message = envelope.message - - envelope_bytes = envelope_pb.SerializeToString() + envelope_bytes = self._serializer.encode(self) return envelope_bytes @classmethod - def decode(cls, envelope_bytes: bytes) -> 'Envelope': + def decode(cls, envelope_bytes: bytes, serializer: Optional[EnvelopeSerializer] = None) -> 'Envelope': """ Decode the envelope. :param envelope_bytes: the bytes to be decoded. + :param serializer: the serializer that implements the decoding procedure. :return: the decoded envelope. """ - envelope_pb = base_pb2.Envelope() - envelope_pb.ParseFromString(envelope_bytes) - - to = envelope_pb.to - sender = envelope_pb.sender - protocol_id = envelope_pb.protocol_id - message = envelope_pb.message - - envelope = Envelope(to=to, sender=sender, - protocol_id=protocol_id, message=message) + if serializer is None: + serializer = DefaultEnvelopeSerializer() + envelope = serializer.decode(envelope_bytes) return envelope def __str__(self): From 704b37ee69dd56f905e6e110fb326bf09de4383f Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 1 Nov 2019 20:03:51 +0100 Subject: [PATCH 28/38] use dependency-injection in the 'encode' method. --- aea/mail/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aea/mail/base.py b/aea/mail/base.py index 34187ab0a0..692c979ec4 100644 --- a/aea/mail/base.py +++ b/aea/mail/base.py @@ -79,8 +79,7 @@ class Envelope: def __init__(self, to: Address, sender: Address, protocol_id: ProtocolId, - message: bytes, - serializer: Optional[EnvelopeSerializer] = None): + message: bytes): """ Initialize a Message object. @@ -88,13 +87,11 @@ def __init__(self, to: Address, :param sender: the public key of the sender. :param protocol_id: the protocol id. :param message: the protocol-specific message - :param serializer: the implementation for the envelope serialization. """ self._to = to self._sender = sender self._protocol_id = protocol_id self._message = message - self._serializer = serializer @property def to(self) -> Address: @@ -144,13 +141,16 @@ def __eq__(self, other): and self.protocol_id == other.protocol_id \ and self._message == other._message - def encode(self) -> bytes: + def encode(self, serializer: Optional[EnvelopeSerializer] = None) -> bytes: """ Encode the envelope. + :param serializer: the serializer that implements the encoding procedure. :return: the encoded envelope. """ - envelope_bytes = self._serializer.encode(self) + if serializer is None: + serializer = DefaultEnvelopeSerializer() + envelope_bytes = serializer.encode(self) return envelope_bytes @classmethod From fd94460d6206f41c18f20ed1fd33fcc8df013342 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Fri, 1 Nov 2019 20:10:00 +0100 Subject: [PATCH 29/38] separate protobuf serializer class from default serializer class. --- aea/mail/base.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/aea/mail/base.py b/aea/mail/base.py index 692c979ec4..d2ef2b85ec 100644 --- a/aea/mail/base.py +++ b/aea/mail/base.py @@ -44,8 +44,8 @@ def decode(self, envelope_bytes: bytes) -> 'Envelope': """Decode the envelope""" -class DefaultEnvelopeSerializer(EnvelopeSerializer): - """Default envelope serializer.""" +class ProtobufEnvelopeSerializer(EnvelopeSerializer): + """Envelope serializer using Protobuf.""" def encode(self, envelope: 'Envelope') -> bytes: """Encode the envelope""" @@ -73,9 +73,14 @@ def decode(self, envelope_bytes: bytes) -> 'Envelope': return envelope +DefaultEnvelopeSerializer = ProtobufEnvelopeSerializer + + class Envelope: """The top level message class.""" + default_serializer = DefaultEnvelopeSerializer() + def __init__(self, to: Address, sender: Address, protocol_id: ProtocolId, @@ -149,7 +154,7 @@ def encode(self, serializer: Optional[EnvelopeSerializer] = None) -> bytes: :return: the encoded envelope. """ if serializer is None: - serializer = DefaultEnvelopeSerializer() + serializer = self.default_serializer envelope_bytes = serializer.encode(self) return envelope_bytes @@ -163,7 +168,7 @@ def decode(cls, envelope_bytes: bytes, serializer: Optional[EnvelopeSerializer] :return: the decoded envelope. """ if serializer is None: - serializer = DefaultEnvelopeSerializer() + serializer = cls.default_serializer envelope = serializer.decode(envelope_bytes) return envelope From 7e8a3e5de068ccb277f35a4c38b31cf6d5c15fca Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 01:41:57 +0100 Subject: [PATCH 30/38] fix code style check. --- aea/mail/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aea/mail/base.py b/aea/mail/base.py index d2ef2b85ec..53a28b09b4 100644 --- a/aea/mail/base.py +++ b/aea/mail/base.py @@ -37,18 +37,18 @@ class EnvelopeSerializer(ABC): @abstractmethod def encode(self, envelope: 'Envelope') -> bytes: - """Encode the envelope""" + """Encode the envelope.""" @abstractmethod def decode(self, envelope_bytes: bytes) -> 'Envelope': - """Decode the envelope""" + """Decode the envelope.""" class ProtobufEnvelopeSerializer(EnvelopeSerializer): """Envelope serializer using Protobuf.""" def encode(self, envelope: 'Envelope') -> bytes: - """Encode the envelope""" + """Encode the envelope.""" envelope_pb = base_pb2.Envelope() envelope_pb.to = envelope.to envelope_pb.sender = envelope.sender @@ -59,7 +59,7 @@ def encode(self, envelope: 'Envelope') -> bytes: return envelope_bytes def decode(self, envelope_bytes: bytes) -> 'Envelope': - """Decode the envelope""" + """Decode the envelope.""" envelope_pb = base_pb2.Envelope() envelope_pb.ParseFromString(envelope_bytes) From 95fc4cfc2ce325ce8a077a8d150a9f5be5dc8438 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 11:45:26 +0100 Subject: [PATCH 31/38] add working prototype of TCP connections. --- aea/connections/p2p/base.py | 173 --------------- aea/connections/p2p/tcp_client.py | 124 ----------- aea/connections/{p2p => tcp}/__init__.py | 2 +- aea/connections/tcp/base.py | 209 +++++++++++++++++++ aea/connections/{p2p => tcp}/connection.py | 2 +- aea/connections/{p2p => tcp}/connection.yaml | 4 +- aea/connections/tcp/tcp_client.py | 102 +++++++++ aea/connections/{p2p => tcp}/tcp_server.py | 94 +++------ 8 files changed, 347 insertions(+), 363 deletions(-) delete mode 100644 aea/connections/p2p/base.py delete mode 100644 aea/connections/p2p/tcp_client.py rename aea/connections/{p2p => tcp}/__init__.py (94%) create mode 100644 aea/connections/tcp/base.py rename aea/connections/{p2p => tcp}/connection.py (95%) rename aea/connections/{p2p => tcp}/connection.yaml (87%) create mode 100644 aea/connections/tcp/tcp_client.py rename aea/connections/{p2p => tcp}/tcp_server.py (55%) diff --git a/aea/connections/p2p/base.py b/aea/connections/p2p/base.py deleted file mode 100644 index 81ea5bdb2b..0000000000 --- a/aea/connections/p2p/base.py +++ /dev/null @@ -1,173 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""Base classes for TCP communication.""" -import logging -import queue -import struct -from abc import ABC -from asyncio import StreamReader, StreamWriter, CancelledError -from queue import Queue -from threading import Thread -from typing import Optional - -from aea.connections.base import Channel, Connection -from aea.mail.base import Envelope - -logger = logging.getLogger(__name__) - - -class TCPChannel(Channel, ABC): - """Abstract TCP channel.""" - - def __init__(self, public_key: str): - """Initialize a TCP Channel.""" - self.in_queue = Queue() # type: Queue - self.public_key = public_key - self._stopped = False - - async def _recv(self, reader: StreamReader) -> Optional[bytes]: - """Receive bytes.""" - try: - data = await reader.read(len(struct.pack("I", 0))) - nbytes = struct.unpack("I", data)[0] - nbytes_read = 0 - data = b"" - while nbytes_read < nbytes: - data += (await reader.read(nbytes - nbytes_read)) - nbytes_read = len(data) - - return data - except CancelledError: - return None - except Exception as e: - logger.exception(e) - return None - - async def _send(self, writer: StreamWriter, data: bytes) -> None: - """Send bytes.""" - logger.debug("Send a message") - nbytes = struct.pack("I", len(data)) - logger.debug("#bytes: {!r}".format(nbytes)) - try: - writer.write(nbytes) - writer.write(data) - await writer.drain() - except CancelledError: - return None - - async def _recv_loop(self, reader) -> None: - """Process incoming messages.""" - try: - if self._stopped: - logger.debug("Stopped receiving loop.") - return - logger.debug("Waiting for next message...") - data = await self._recv(reader) - if data is None: - return - logger.debug("Message received: {!r}".format(data)) - envelope = Envelope.decode(data) # TODO handle decoding error - logger.debug("Decoded envelope: {}".format(envelope)) - self.in_queue.put_nowait(envelope) - await self._recv_loop(reader) - except Exception as e: - logger.exception(e) - return - - def recv(self, block=True, timeout=None) -> Optional[Envelope]: - """Receive an envelope.""" - try: - return self.in_queue.get(block=block, timeout=timeout) - except queue.Empty: - return None - - -class TCPConnection(Connection, ABC): - """Abstract TCP connection.""" - - _channel: TCPChannel - - def __init__(self, public_key: str, channel: TCPChannel): - """ - Initialize a TCP connection. - - :param public_key: the public key. - :param channel: the TCP channel. - """ - super().__init__() - - self.public_key = public_key - self._channel = channel - self._stopped = True - - def _fetch(self) -> None: - """Fetch the envelopes from the outqueue and send them.""" - while not self._stopped: - try: - msg = self.out_queue.get(block=True, timeout=2.0) - if msg is not None: - self.send(msg) - except queue.Empty: - pass - - def _receive_loop(self): - """Receive envelopes.""" - while not self._stopped: - try: - data = self._channel.recv(block=True, timeout=2.0) - if data is not None: - self.in_queue.put_nowait(data) - except queue.Empty: - pass - - @property - def is_established(self) -> bool: - """Return True if the connection has been established, False otherwise.""" - return not self._stopped - - def connect(self): - """Connect to the local OEF Node.""" - if self._stopped: - self._stopped = False - self._channel.connect() - self.in_thread = Thread(target=self._receive_loop) - self.out_thread = Thread(target=self._fetch) - self.in_thread.start() - self.out_thread.start() - - def disconnect(self): - """Disconnect from the local OEF Node.""" - if not self._stopped: - self._stopped = True - self._channel.disconnect() - self.in_thread.join() - self.out_thread.join() - self.in_thread = None - self.out_thread = None - - def send(self, envelope: Envelope): - """Send a message.""" - if not self.is_established: - raise ConnectionError("Connection not established yet. Please use 'connect()'.") - self._channel.send(envelope) - - def stop(self): - """Tear down the connection.""" - self._channel.disconnect() diff --git a/aea/connections/p2p/tcp_client.py b/aea/connections/p2p/tcp_client.py deleted file mode 100644 index e82a2d2a02..0000000000 --- a/aea/connections/p2p/tcp_client.py +++ /dev/null @@ -1,124 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""Implementation of the TCP server.""" -import asyncio -import logging -from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader -from threading import Thread -from typing import Optional, cast - -from aea.configurations.base import ConnectionConfig -from aea.connections.base import Connection -from aea.connections.p2p.base import TCPChannel, TCPConnection -from aea.mail.base import Envelope - -logger = logging.getLogger(__name__) - -STUB_DIALOGUE_ID = 0 - - -class TCPClientChannel(TCPChannel): - """Channel implementation for the local node.""" - - def __init__(self, public_key: str, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): - """ - Initialize a TCP channel. - - :param public_key: the public key. - :param address: the socket bind address. - :param unix: whether it's a unix server or a networked. - :param loop: the asyncio loop. - """ - super().__init__(public_key) - self.address = address - self.unix = unix - self._loop = asyncio.new_event_loop() if loop is None else loop - self._thread = None # type: Optional[Thread] - - self._reader, self._writer = (None, None) # type: Optional[StreamReader], Optional[StreamWriter] - self._read_task = None # type: Optional[Task] - - def connect(self): - """ - Set up the connection. - - :return: A queue or None. - """ - self._thread = Thread(target=self._loop.run_forever) - self._thread.start() - - if self.unix: - coro = asyncio.open_unix_connection(path=self.address, loop=self._loop) - else: - ip, port = self.address.split(":") - port = int(port) - coro = asyncio.open_connection(ip, port, loop=self._loop, start_serving=False) - - future = asyncio.run_coroutine_threadsafe(coro, self._loop) - self._reader, self._writer = future.result() - - public_key_bytes = self.public_key.encode("utf-8") - future = asyncio.run_coroutine_threadsafe(self._send(self._writer, public_key_bytes), loop=self._loop) - future.result() - - asyncio.run_coroutine_threadsafe(self._recv_loop(self._reader), self._loop) - - def disconnect(self) -> None: - """ - Tear down the connection. - - :return: None. - """ - if self._stopped: - return - - self._stopped = True - self._writer = cast(StreamWriter, self._writer) - self._thread = cast(Thread, self._thread) - - self._writer.close() - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread.join() - - def send(self, envelope: Envelope) -> None: - """ - Send an envelope. - - :param envelope: the envelope to send. - :return: None. - """ - self._writer = cast(StreamWriter, self._writer) - asyncio.run_coroutine_threadsafe(self._send(self._writer, envelope.encode()), loop=self._loop) - - -class TCPClientConnection(TCPConnection): - """Implementation of a TCP server connection.""" - - @classmethod - def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': - """Get the Local OEF connection from the connection configuration. - - :param public_key: the public key of the agent. - :param connection_configuration: the connection configuration object. - :return: the connection object - """ - address = cast(str, connection_configuration.config.get("address")) - channel = TCPClientChannel(public_key, address, unix=True) - return TCPClientConnection(public_key, channel) diff --git a/aea/connections/p2p/__init__.py b/aea/connections/tcp/__init__.py similarity index 94% rename from aea/connections/p2p/__init__.py rename to aea/connections/tcp/__init__.py index 21d4686b60..c001f65a6f 100644 --- a/aea/connections/p2p/__init__.py +++ b/aea/connections/tcp/__init__.py @@ -17,4 +17,4 @@ # # ------------------------------------------------------------------------------ -"""Implementation of a P2P connection.""" +"""Implementation of a TCP connection.""" diff --git a/aea/connections/tcp/base.py b/aea/connections/tcp/base.py new file mode 100644 index 0000000000..cb687935e7 --- /dev/null +++ b/aea/connections/tcp/base.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Base classes for TCP communication.""" +import asyncio +import logging +import struct +import threading +from abc import ABC, abstractmethod +from asyncio import CancelledError, StreamWriter, StreamReader, Queue, transports, AbstractEventLoop +from threading import Thread +from typing import Optional, Protocol + +from aea.connections.base import Connection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + + +class AEAStreamProtocol(Protocol): + """A simple stream protocol for AEAs.""" + + def __init__(self, queue: Queue): + """ + Initialize a protocol object. + + :param queue: the queue that stores incoming messages. + """ + self.queue = queue + + def connection_made(self, transport: transports.BaseTransport) -> None: + """Validate the connection made.""" + + def data_received(self, data: bytes) -> None: + """Handle the received data.""" + + def eof_received(self) -> Optional[bool]: + """Handle the end of the connection.""" + + +class TCPConnection(Connection, ABC): + """Abstract TCP connection.""" + + def __init__(self, + public_key: str, + host: str, + port: int, + unix: bool = True, + loop: Optional[AbstractEventLoop] = None): + """Initialize the TCP connection.""" + super().__init__() + self.public_key = public_key + + self.host = host + self.port = port + self.unix = unix + self._loop = asyncio.new_event_loop() if loop is None else loop + + self._lock = threading.Lock() + self._stopped = True + self._connected = False + self._thread_loop = None # type: Optional[Thread] + + def _run_task(self, coro): + assert self._loop.is_running() + return asyncio.run_coroutine_threadsafe(coro=coro, loop=self._loop) + + @property + def _is_threaded(self) -> bool: + """Check if the loop is run by our thread or from another thread.""" + return self._loop.is_running() and self._thread_loop is None + + def _start_loop(self): + assert self._thread_loop is None + self._thread_loop = Thread(target=self._loop.run_forever) + self._thread_loop.start() + + def _stop_loop(self): + assert self._thread_loop.is_alive() + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread_loop.join(timeout=10) + self._thread_loop = None + + @abstractmethod + def setup(self): + """Set the TCP connection up.""" + + @abstractmethod + def teardown(self): + """Tear the TCP connection down.""" + + @abstractmethod + def select_writer_from_envelope(self, envelope: Envelope) -> StreamWriter: + """Select the destination, given the envelope""" + + @property + def is_established(self): + """Check if the connection is established.""" + return not self._stopped and self._connected + + def connect(self): + """ + Set up the connection. + + :return: A queue or None. + """ + with self._lock: + if self.is_established: + logger.warning("Connection already set up.") + return + + self._stopped = False + if not self._is_threaded: + self._start_loop() + self.setup() + self._connected = True + + def disconnect(self) -> None: + """ + Tear down the connection. + + :return: None. + """ + with self._lock: + if not self.is_established: + logger.warning("Connection is not set up.") + return + + self._connected = False + self.teardown() + if not self._is_threaded: + self._stop_loop() + self._stopped = True + + async def _recv(self, reader: StreamReader) -> Optional[bytes]: + """Receive bytes.""" + try: + data = await reader.read(len(struct.pack("I", 0))) + nbytes = struct.unpack("I", data)[0] + nbytes_read = 0 + data = b"" + while nbytes_read < nbytes: + data += (await reader.read(nbytes - nbytes_read)) + nbytes_read = len(data) + + return data + except CancelledError: + return None + except Exception as e: + logger.exception(e) + return None + + async def _send(self, writer: StreamWriter, data: bytes) -> None: + """Send bytes.""" + logger.debug("Send a message") + nbytes = struct.pack("I", len(data)) + logger.debug("#bytes: {!r}".format(nbytes)) + try: + writer.write(nbytes) + writer.write(data) + await writer.drain() + except CancelledError: + return None + + async def _recv_loop(self, reader) -> None: + """Process incoming messages.""" + try: + if not self.is_established: + logger.debug("Stopped receiving loop.") + return + logger.debug("Waiting for next message...") + data = await self._recv(reader) + if data is None: + return + logger.debug("Message received: {!r}".format(data)) + envelope = Envelope.decode(data) # TODO handle decoding error + logger.debug("Decoded envelope: {}".format(envelope)) + self.in_queue.put_nowait(envelope) + await self._recv_loop(reader) + except Exception as e: + logger.exception(e) + return + + def send(self, envelope: Envelope) -> None: + """ + Send an envelope. + + :param envelope: the envelope to send. + :return: None. + """ + writer = self.select_writer_from_envelope(envelope) + future = self._run_task(self._send(writer, envelope.encode())) + future.result() # TODO avoid waiting and handle cancellation \ No newline at end of file diff --git a/aea/connections/p2p/connection.py b/aea/connections/tcp/connection.py similarity index 95% rename from aea/connections/p2p/connection.py rename to aea/connections/tcp/connection.py index 08c185e93b..59137f343a 100644 --- a/aea/connections/p2p/connection.py +++ b/aea/connections/tcp/connection.py @@ -18,7 +18,7 @@ # # ------------------------------------------------------------------------------ -"""Base classes for P2P communication.""" +"""Base classes for TCP communication.""" from .tcp_client import TCPClientConnection # noqa: F401 from .tcp_server import TCPServerConnection # noqa: F401 diff --git a/aea/connections/p2p/connection.yaml b/aea/connections/tcp/connection.yaml similarity index 87% rename from aea/connections/p2p/connection.yaml rename to aea/connections/tcp/connection.yaml index 8ff67f76ba..b2e031ec61 100644 --- a/aea/connections/p2p/connection.yaml +++ b/aea/connections/tcp/connection.yaml @@ -11,5 +11,5 @@ supported_protocols: - gym - tac config: - address: /tmp/socket - unix: true \ No newline at end of file + address: 127.0.0.1 + port: 8082 \ No newline at end of file diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py new file mode 100644 index 0000000000..44f5b080a1 --- /dev/null +++ b/aea/connections/tcp/tcp_client.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the TCP server.""" +import asyncio +import logging +from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader, AbstractServer +from threading import Thread, Lock +from typing import Optional, cast, Tuple, Dict + +from aea.configurations.base import ConnectionConfig +from aea.connections.base import Connection +from aea.connections.tcp.base import TCPConnection +from aea.mail.base import Envelope + +logger = logging.getLogger(__name__) + +STUB_DIALOGUE_ID = 0 + + +class TCPClientConnection(TCPConnection): + """Abstract TCP channel.""" + + def __init__(self, + public_key: str, + host: str, + port: int, + unix: bool = True, + loop: Optional[AbstractEventLoop] = None): + """ + Initialize a TCP channel. + + :param public_key: public key. + :param host: the socket bind address. + :param unix: whether a UNIX socket is used. + :param loop: the asyncio loop. + """ + super().__init__(public_key, host, port, unix=unix, loop=loop) + + self._reader, self._writer = (None, None) # type: Optional[StreamReader], Optional[StreamWriter] + self._read_task = None # type: Optional[Task] + + def _run_task(self, coro): + assert self._loop.is_running() + return asyncio.run_coroutine_threadsafe(coro=coro, loop=self._loop) + + def _start_loop(self): + assert self._thread_loop is None + self._thread_loop = Thread(target=self._loop.run_forever) + self._thread_loop.start() + + def _stop_loop(self): + assert self._thread_loop.is_alive() + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread_loop.join(timeout=10) + self._thread_loop = None + + def setup(self): + """Set the connection up.""" + future = self._run_task(coro=asyncio.open_connection(self.host, self.port, loop=self._loop)) + self._reader, self._writer = future.result() + public_key_bytes = self.public_key.encode("utf-8") + future = self._run_task(self._send(self._writer, public_key_bytes)) + future.result() + self._read_task = self._run_task(self._recv_loop(self._reader)) # TODO store future to handle cancellation + + def teardown(self): + """Tear the connection down.""" + self._writer.close() + self._read_task.result() + + def select_writer_from_envelope(self, envelope: Envelope): + """Select the destination, given the envelope.""" + return self._writer + + @classmethod + def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': + """Get the TCP server connection from the connection configuration. + + :param public_key: the public key of the agent. + :param connection_configuration: the connection configuration object. + :return: the connection object + """ + address = cast(str, connection_configuration.config.get("address")) + port = cast(int, connection_configuration.config.get("port")) + return TCPClientConnection(public_key, address, port) diff --git a/aea/connections/p2p/tcp_server.py b/aea/connections/tcp/tcp_server.py similarity index 55% rename from aea/connections/p2p/tcp_server.py rename to aea/connections/tcp/tcp_server.py index 69ece6c081..6788c88cbe 100644 --- a/aea/connections/p2p/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -20,14 +20,18 @@ """Implementation of the TCP server.""" import asyncio +import functools import logging -from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer -from threading import Thread +import struct +from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, Protocol, transports, \ + CancelledError +from queue import Queue +from threading import Thread, Lock from typing import Dict, Optional, Tuple, cast from aea.configurations.base import ConnectionConfig from aea.connections.base import Connection -from aea.connections.p2p.base import TCPChannel, TCPConnection +from aea.connections.tcp.base import TCPConnection from aea.mail.base import Envelope logger = logging.getLogger(__name__) @@ -35,27 +39,27 @@ STUB_DIALOGUE_ID = 0 -class TCPServerChannel(TCPChannel): - """Channel implementation for the local node.""" - - def __init__(self, public_key: str, address: str, unix: bool = True, loop: Optional[AbstractEventLoop] = None): +class TCPServerConnection(TCPConnection): + """Abstract TCP channel.""" + + def __init__(self, + public_key: str, + host: str, + port: int, + unix: bool = True, + loop: Optional[AbstractEventLoop] = None): """ Initialize a TCP channel. :param public_key: public key. - :param address: the socket bind address. - :param unix: whether it's a unix server or a networked. + :param host: the socket bind address. + :param unix: whether a UNIX socket is used. :param loop: the asyncio loop. """ - super().__init__(public_key) - self.address = address - self.unix = unix - self._loop = asyncio.new_event_loop() if loop is None else loop + super().__init__(public_key, host, port, unix=unix, loop=loop) - self._thread = None # type: Optional[Thread] self._server = None # type: Optional[AbstractServer] self._server_task = None # type: Optional[Task] - self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: @@ -75,65 +79,31 @@ async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: self.connections[public_key] = (reader, writer) await self._recv_loop(reader) - def connect(self): - """ - Set up the connection. - - :return: A queue or None. - """ - self._thread = Thread(target=self._loop.run_forever) - self._thread.start() - - if self.unix: - coro = asyncio.start_unix_server(self.handle, self.address, loop=self._loop, start_serving=False) - else: - coro = asyncio.start_server(self.handle, self.address, loop=self._loop, start_serving=False) - future = asyncio.run_coroutine_threadsafe(coro, self._loop) + def setup(self): + """Set the connection up.""" + future = self._run_task(asyncio.start_server(self.handle, host=self.host, port=self.port, + loop=self._loop, start_serving=False)) self._server = future.result() - self._server_task = asyncio.run_coroutine_threadsafe(self._server.serve_forever(), loop=self._loop) - - def disconnect(self) -> None: - """ - Tear down the connection. - - :return: None. - """ - if self._stopped: - return - - self._stopped = True - - self._server = cast(AbstractServer, self._server) - self._thread = cast(Thread, self._thread) + self._server_task = self._run_task(self._server.serve_forever()) + def teardown(self): + """Tear the connection down.""" self._server.close() - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread.join() - - def send(self, envelope: Envelope) -> None: - """ - Send an envelope. - :param envelope: the envelope to send. - :return: None. - """ + def select_writer_from_envelope(self, envelope: Envelope): + """Select the destination, given the envelope.""" to = envelope.to _, writer = self.connections[to] - future = asyncio.run_coroutine_threadsafe(self._send(writer, envelope.encode()), loop=self._loop) - future.result() - - -class TCPServerConnection(TCPConnection): - """Implementation of a TCP server connection.""" + return writer @classmethod def from_config(cls, public_key: str, connection_configuration: ConnectionConfig) -> 'Connection': - """Get the Local OEF connection from the connection configuration. + """Get the TCP server connection from the connection configuration. :param public_key: the public key of the agent. :param connection_configuration: the connection configuration object. :return: the connection object """ address = cast(str, connection_configuration.config.get("address")) - channel = TCPServerChannel(public_key, address, unix=True) - return TCPServerConnection(public_key, channel) + port = cast(int, connection_configuration.config.get("port")) + return TCPServerConnection(public_key, address, port) From d63665237eeb84ae1ba65db103a584e39ce611b6 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 11:48:21 +0100 Subject: [PATCH 32/38] remove unix flag parameter to TCPConnection constructor. --- aea/connections/tcp/base.py | 2 -- aea/connections/tcp/tcp_client.py | 4 +--- aea/connections/tcp/tcp_server.py | 6 ++---- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/aea/connections/tcp/base.py b/aea/connections/tcp/base.py index cb687935e7..82b718463b 100644 --- a/aea/connections/tcp/base.py +++ b/aea/connections/tcp/base.py @@ -61,7 +61,6 @@ def __init__(self, public_key: str, host: str, port: int, - unix: bool = True, loop: Optional[AbstractEventLoop] = None): """Initialize the TCP connection.""" super().__init__() @@ -69,7 +68,6 @@ def __init__(self, self.host = host self.port = port - self.unix = unix self._loop = asyncio.new_event_loop() if loop is None else loop self._lock = threading.Lock() diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py index 44f5b080a1..fb951106aa 100644 --- a/aea/connections/tcp/tcp_client.py +++ b/aea/connections/tcp/tcp_client.py @@ -41,17 +41,15 @@ def __init__(self, public_key: str, host: str, port: int, - unix: bool = True, loop: Optional[AbstractEventLoop] = None): """ Initialize a TCP channel. :param public_key: public key. :param host: the socket bind address. - :param unix: whether a UNIX socket is used. :param loop: the asyncio loop. """ - super().__init__(public_key, host, port, unix=unix, loop=loop) + super().__init__(public_key, host, port, loop=loop) self._reader, self._writer = (None, None) # type: Optional[StreamReader], Optional[StreamWriter] self._read_task = None # type: Optional[Task] diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index 6788c88cbe..0bd4a4a8d5 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -46,17 +46,15 @@ def __init__(self, public_key: str, host: str, port: int, - unix: bool = True, loop: Optional[AbstractEventLoop] = None): """ Initialize a TCP channel. :param public_key: public key. :param host: the socket bind address. - :param unix: whether a UNIX socket is used. - :param loop: the asyncio loop. + :param loop: the asyncio loop. """ - super().__init__(public_key, host, port, unix=unix, loop=loop) + super().__init__(public_key, host, port, loop=loop) self._server = None # type: Optional[AbstractServer] self._server_task = None # type: Optional[Task] From f0c6da9784c7b94a6de0220006fc1ab099ba97e2 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 17:22:22 +0100 Subject: [PATCH 33/38] fix tcp connection implementation and add tests. --- aea/connections/tcp/base.py | 100 +++++++++++------- aea/connections/tcp/tcp_client.py | 17 +-- aea/connections/tcp/tcp_server.py | 16 +-- tests/test_connections/test_p2p.py | 63 ----------- tests/test_connections/test_tcp/__init__.py | 20 ++++ .../test_tcp/test_communication.py | 85 +++++++++++++++ 6 files changed, 184 insertions(+), 117 deletions(-) delete mode 100644 tests/test_connections/test_p2p.py create mode 100644 tests/test_connections/test_tcp/__init__.py create mode 100644 tests/test_connections/test_tcp/test_communication.py diff --git a/aea/connections/tcp/base.py b/aea/connections/tcp/base.py index 82b718463b..40701318ee 100644 --- a/aea/connections/tcp/base.py +++ b/aea/connections/tcp/base.py @@ -22,10 +22,12 @@ import logging import struct import threading +from _queue import Empty from abc import ABC, abstractmethod -from asyncio import CancelledError, StreamWriter, StreamReader, Queue, transports, AbstractEventLoop +from asyncio import CancelledError, StreamWriter, StreamReader, AbstractEventLoop, Task +from concurrent.futures import Executor from threading import Thread -from typing import Optional, Protocol +from typing import Optional from aea.connections.base import Connection from aea.mail.base import Envelope @@ -33,27 +35,6 @@ logger = logging.getLogger(__name__) -class AEAStreamProtocol(Protocol): - """A simple stream protocol for AEAs.""" - - def __init__(self, queue: Queue): - """ - Initialize a protocol object. - - :param queue: the queue that stores incoming messages. - """ - self.queue = queue - - def connection_made(self, transport: transports.BaseTransport) -> None: - """Validate the connection made.""" - - def data_received(self, data: bytes) -> None: - """Handle the received data.""" - - def eof_received(self) -> Optional[bool]: - """Handle the end of the connection.""" - - class TCPConnection(Connection, ABC): """Abstract TCP connection.""" @@ -61,7 +42,8 @@ def __init__(self, public_key: str, host: str, port: int, - loop: Optional[AbstractEventLoop] = None): + loop: Optional[AbstractEventLoop] = None, + executor: Optional[Executor] = None): """Initialize the TCP connection.""" super().__init__() self.public_key = public_key @@ -74,9 +56,10 @@ def __init__(self, self._stopped = True self._connected = False self._thread_loop = None # type: Optional[Thread] + self._recv_task = None # type: Optional[Task] + self._fetch_task = None # type: Optional[Task] def _run_task(self, coro): - assert self._loop.is_running() return asyncio.run_coroutine_threadsafe(coro=coro, loop=self._loop) @property @@ -86,7 +69,13 @@ def _is_threaded(self) -> bool: def _start_loop(self): assert self._thread_loop is None - self._thread_loop = Thread(target=self._loop.run_forever) + + def loop_in_thread(loop): + asyncio.set_event_loop(loop) + loop.run_forever() + loop.close() + + self._thread_loop = Thread(target=loop_in_thread, args=(self._loop, )) self._thread_loop.start() def _stop_loop(self): @@ -104,8 +93,13 @@ def teardown(self): """Tear the TCP connection down.""" @abstractmethod - def select_writer_from_envelope(self, envelope: Envelope) -> StreamWriter: - """Select the destination, given the envelope""" + def select_writer_from_envelope(self, envelope: Envelope) -> Optional[StreamWriter]: + """ + Select the destination, given the envelope + + :param envelope: the envelope to be sent. + :return: the stream writer to communicate with the recipient. None if it cannot be determined. + """ @property def is_established(self): @@ -141,6 +135,8 @@ def disconnect(self) -> None: return self._connected = False + self.out_queue.put(None) + self._fetch_task.result() self.teardown() if not self._is_threaded: self._stop_loop() @@ -150,23 +146,27 @@ async def _recv(self, reader: StreamReader) -> Optional[bytes]: """Receive bytes.""" try: data = await reader.read(len(struct.pack("I", 0))) + if not self._connected: + return nbytes = struct.unpack("I", data)[0] nbytes_read = 0 data = b"" while nbytes_read < nbytes: data += (await reader.read(nbytes - nbytes_read)) nbytes_read = len(data) - return data except CancelledError: - return None + logger.debug("[{}] Read cancelled.".format(self.public_key)) + raise + except struct.error as e: + logger.debug("Struct error: {}".format(str(e))) except Exception as e: logger.exception(e) - return None + raise async def _send(self, writer: StreamWriter, data: bytes) -> None: """Send bytes.""" - logger.debug("Send a message") + logger.debug("[{}] Send a message".format(self.public_key)) nbytes = struct.pack("I", len(data)) logger.debug("#bytes: {!r}".format(nbytes)) try: @@ -179,18 +179,38 @@ async def _send(self, writer: StreamWriter, data: bytes) -> None: async def _recv_loop(self, reader) -> None: """Process incoming messages.""" try: - if not self.is_established: - logger.debug("Stopped receiving loop.") - return - logger.debug("Waiting for next message...") + logger.debug("[{}]: Waiting for receiving next message...".format(self.public_key)) data = await self._recv(reader) if data is None: return - logger.debug("Message received: {!r}".format(data)) + logger.debug("[{}] Message received: {!r}".format(self.public_key, data)) envelope = Envelope.decode(data) # TODO handle decoding error - logger.debug("Decoded envelope: {}".format(envelope)) + logger.debug("[{}] Decoded envelope: {}".format(self.public_key, envelope)) self.in_queue.put_nowait(envelope) await self._recv_loop(reader) + except CancelledError: + logger.debug("[{}] Receiving loop cancelled.".format(self.public_key)) + return + except Exception as e: + logger.exception(e) + return + + async def _send_loop(self): + """Process outgoing messages.""" + try: + logger.debug("[{}]: Waiting for sending next message...".format(self.public_key)) + envelope = await self._loop.run_in_executor(None, self.out_queue.get, True) + if envelope is None: + logger.debug("[{}] Stopped sending loop.".format(self.public_key)) + return + writer = self.select_writer_from_envelope(envelope) + await self._send(writer, envelope.encode()) + await self._send_loop() + except CancelledError: + logger.debug("[{}] Sending loop cancelled.".format(self.public_key)) + return + except Empty: + await self._send_loop() except Exception as e: logger.exception(e) return @@ -202,6 +222,4 @@ def send(self, envelope: Envelope) -> None: :param envelope: the envelope to send. :return: None. """ - writer = self.select_writer_from_envelope(envelope) - future = self._run_task(self._send(writer, envelope.encode())) - future.result() # TODO avoid waiting and handle cancellation \ No newline at end of file + self.out_queue.put(envelope) diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py index fb951106aa..a2f2f9efbb 100644 --- a/aea/connections/tcp/tcp_client.py +++ b/aea/connections/tcp/tcp_client.py @@ -20,9 +20,10 @@ """Implementation of the TCP server.""" import asyncio import logging -from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader, AbstractServer -from threading import Thread, Lock -from typing import Optional, cast, Tuple, Dict +from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader +from concurrent.futures import CancelledError +from threading import Thread +from typing import Optional, cast from aea.configurations.base import ConnectionConfig from aea.connections.base import Connection @@ -71,19 +72,23 @@ def _stop_loop(self): def setup(self): """Set the connection up.""" - future = self._run_task(coro=asyncio.open_connection(self.host, self.port, loop=self._loop)) + future = self._run_task(asyncio.open_connection(self.host, self.port, loop=self._loop)) self._reader, self._writer = future.result() public_key_bytes = self.public_key.encode("utf-8") future = self._run_task(self._send(self._writer, public_key_bytes)) future.result() self._read_task = self._run_task(self._recv_loop(self._reader)) # TODO store future to handle cancellation + self._fetch_task = self._run_task(self._send_loop()) def teardown(self): """Tear the connection down.""" + try: + self._read_task.cancel() + except CancelledError: + pass self._writer.close() - self._read_task.result() - def select_writer_from_envelope(self, envelope: Envelope): + def select_writer_from_envelope(self, envelope: Envelope) -> Optional[StreamWriter]: """Select the destination, given the envelope.""" return self._writer diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index 0bd4a4a8d5..d4cd4af8ab 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -20,13 +20,8 @@ """Implementation of the TCP server.""" import asyncio -import functools import logging -import struct -from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer, Protocol, transports, \ - CancelledError -from queue import Queue -from threading import Thread, Lock +from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer from typing import Dict, Optional, Tuple, cast from aea.configurations.base import ConnectionConfig @@ -59,6 +54,7 @@ def __init__(self, self._server = None # type: Optional[AbstractServer] self._server_task = None # type: Optional[Task] self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] + self._read_tasks = set() async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: """ @@ -75,7 +71,8 @@ async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: public_key = public_key_bytes.decode("utf-8") logger.debug("Public key of the client: {}".format(public_key)) self.connections[public_key] = (reader, writer) - await self._recv_loop(reader) + task = self._loop.create_task(self._recv_loop(reader)) + self._read_tasks.add(task) def setup(self): """Set the connection up.""" @@ -83,14 +80,19 @@ def setup(self): loop=self._loop, start_serving=False)) self._server = future.result() self._server_task = self._run_task(self._server.serve_forever()) + self._fetch_task = self._run_task(self._send_loop()) def teardown(self): """Tear the connection down.""" + for t in self._read_tasks: + t.cancel() self._server.close() def select_writer_from_envelope(self, envelope: Envelope): """Select the destination, given the envelope.""" to = envelope.to + if to not in self.connections: + return None _, writer = self.connections[to] return writer diff --git a/tests/test_connections/test_p2p.py b/tests/test_connections/test_p2p.py deleted file mode 100644 index 76bcf7669c..0000000000 --- a/tests/test_connections/test_p2p.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ - -"""This module contains the tests of the P2P channel.""" - -# class TestTCP: -# """Test TCP connections.""" -# -# @classmethod -# def setup_class(cls): -# """Set up the test class.""" -# p = Path("/tmp/aea/test_tcp/") -# shutil.rmtree(str(p), ignore_errors=True) -# p.mkdir(parents=True) -# -# socket_path = str(Path(p, "test_socket")) -# cls.server_pbk, cls.client_pbk = "server_pbk", "client_pbk" -# server_conn = TCPServerConnection(cls.server_pbk, TCPServerChannel(cls.server_pbk, socket_path, unix=True)) -# client_conn = TCPClientConnection(cls.client_pbk, TCPClientChannel(cls.client_pbk, socket_path, unix=True)) -# -# cls.server_mailbox = MailBox(server_conn) -# cls.client_mailbox = MailBox(client_conn) -# -# cls.server_mailbox.connect() -# cls.client_mailbox.connect() -# -# def test_communication(self): -# """Test that we are able to send an envelope from client to server.""" -# msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello server") -# expected_envelope = Envelope(to=self.server_pbk, sender=self.client_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) -# self.client_mailbox.outbox.put(expected_envelope) -# actual_envelope = self.server_mailbox.inbox.get(timeout=2.0) -# logging.debug(actual_envelope) -# assert expected_envelope == actual_envelope -# -# msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello client") -# expected_envelope = Envelope(to=self.client_pbk, sender=self.server_pbk, protocol_id="default", message=DefaultSerializer().encode(msg)) -# self.server_mailbox.outbox.put(expected_envelope) -# actual_envelope = self.client_mailbox.inbox.get(timeout=30.0) -# logging.debug(actual_envelope) -# assert expected_envelope == actual_envelope -# -# @classmethod -# def teardown_class(cls): -# """Tear down the test class.""" -# cls.server_mailbox.disconnect() -# cls.client_mailbox.disconnect() diff --git a/tests/test_connections/test_tcp/__init__.py b/tests/test_connections/test_tcp/__init__.py new file mode 100644 index 0000000000..1c384a7676 --- /dev/null +++ b/tests/test_connections/test_tcp/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests of the TCP connection.""" diff --git a/tests/test_connections/test_tcp/test_communication.py b/tests/test_connections/test_tcp/test_communication.py new file mode 100644 index 0000000000..459175ca76 --- /dev/null +++ b/tests/test_connections/test_tcp/test_communication.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the TCP connection communication.""" +from aea.connections.tcp.tcp_client import TCPClientConnection +from aea.connections.tcp.tcp_server import TCPServerConnection +from aea.mail.base import MailBox, Envelope +from aea.protocols.default.message import DefaultMessage +from aea.protocols.default.serialization import DefaultSerializer + + +class TestTCPCommunication: + """Test that TCP Server and TCP Client can communicate.""" + + @classmethod + def setup_class(cls): + """Set up the test class.""" + + cls.host = "127.0.0.1" + cls.port = 8082 + + cls.server_pbk = "server_pbk" + cls.client_pbk_1 = "client_pbk_1" + cls.client_pbk_2 = "client_pbk_2" + cls.server_conn = TCPServerConnection(cls.server_pbk, cls.host, cls.port) + cls.client_conn_1 = TCPClientConnection(cls.client_pbk_1, cls.host, cls.port) + cls.client_conn_2 = TCPClientConnection(cls.client_pbk_2, cls.host, cls.port) + + cls.server_mailbox = MailBox(cls.server_conn) + cls.client_1_mailbox = MailBox(cls.client_conn_1) + cls.client_2_mailbox = MailBox(cls.client_conn_2) + + cls.server_mailbox.connect() + cls.client_1_mailbox.connect() + cls.client_2_mailbox.connect() + + def test_communication_client_server(self): + """Test that envelopes can be sent from a client to a server.""" + msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello") + msg_bytes = DefaultSerializer().encode(msg) + expected_envelope = Envelope(to=self.server_pbk, sender=self.client_pbk_1, protocol_id=DefaultMessage.protocol_id, message=msg_bytes) + self.client_1_mailbox.outbox.put(expected_envelope) + actual_envelope = self.server_mailbox.inbox.get(block=True, timeout=5.0) + + assert expected_envelope == actual_envelope + + def test_communication_server_client(self): + """Test that envelopes can be sent from a server to a client.""" + msg = DefaultMessage(type=DefaultMessage.Type.BYTES, content=b"hello") + msg_bytes = DefaultSerializer().encode(msg) + + expected_envelope = Envelope(to=self.client_pbk_1, sender=self.server_pbk, protocol_id=DefaultMessage.protocol_id, message=msg_bytes) + self.server_mailbox.outbox.put(expected_envelope) + actual_envelope = self.client_1_mailbox.inbox.get(block=True, timeout=5.0) + + assert expected_envelope == actual_envelope + + expected_envelope = Envelope(to=self.client_pbk_2, sender=self.server_pbk, protocol_id=DefaultMessage.protocol_id, message=msg_bytes) + self.server_mailbox.outbox.put(expected_envelope) + actual_envelope = self.client_2_mailbox.inbox.get(block=True, timeout=5.0) + + assert expected_envelope == actual_envelope + + @classmethod + def teardown_class(cls): + """Tear down the test class.""" + cls.server_mailbox.disconnect() + cls.client_1_mailbox.disconnect() + cls.client_2_mailbox.disconnect() From 908c09ceb7eb8345ccea08461497f4f167bac7a3 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 17:31:43 +0100 Subject: [PATCH 34/38] fix code checks. --- aea/connections/tcp/base.py | 12 ++++++------ aea/connections/tcp/tcp_client.py | 2 ++ aea/connections/tcp/tcp_server.py | 6 ++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/aea/connections/tcp/base.py b/aea/connections/tcp/base.py index 40701318ee..91617a1325 100644 --- a/aea/connections/tcp/base.py +++ b/aea/connections/tcp/base.py @@ -20,9 +20,9 @@ """Base classes for TCP communication.""" import asyncio import logging +import queue import struct import threading -from _queue import Empty from abc import ABC, abstractmethod from asyncio import CancelledError, StreamWriter, StreamReader, AbstractEventLoop, Task from concurrent.futures import Executor @@ -95,7 +95,7 @@ def teardown(self): @abstractmethod def select_writer_from_envelope(self, envelope: Envelope) -> Optional[StreamWriter]: """ - Select the destination, given the envelope + Select the destination, given the envelope. :param envelope: the envelope to be sent. :return: the stream writer to communicate with the recipient. None if it cannot be determined. @@ -135,8 +135,6 @@ def disconnect(self) -> None: return self._connected = False - self.out_queue.put(None) - self._fetch_task.result() self.teardown() if not self._is_threaded: self._stop_loop() @@ -147,7 +145,7 @@ async def _recv(self, reader: StreamReader) -> Optional[bytes]: try: data = await reader.read(len(struct.pack("I", 0))) if not self._connected: - return + return None nbytes = struct.unpack("I", data)[0] nbytes_read = 0 data = b"" @@ -163,6 +161,8 @@ async def _recv(self, reader: StreamReader) -> Optional[bytes]: except Exception as e: logger.exception(e) raise + finally: + return None async def _send(self, writer: StreamWriter, data: bytes) -> None: """Send bytes.""" @@ -209,7 +209,7 @@ async def _send_loop(self): except CancelledError: logger.debug("[{}] Sending loop cancelled.".format(self.public_key)) return - except Empty: + except queue.Empty: await self._send_loop() except Exception as e: logger.exception(e) diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py index a2f2f9efbb..1f871a7bcc 100644 --- a/aea/connections/tcp/tcp_client.py +++ b/aea/connections/tcp/tcp_client.py @@ -83,6 +83,8 @@ def setup(self): def teardown(self): """Tear the connection down.""" try: + self.out_queue.put(None) + self._fetch_task.result() self._read_task.cancel() except CancelledError: pass diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index d4cd4af8ab..8b9834cc52 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -22,7 +22,7 @@ import asyncio import logging from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer -from typing import Dict, Optional, Tuple, cast +from typing import Dict, Optional, Tuple, cast, Set from aea.configurations.base import ConnectionConfig from aea.connections.base import Connection @@ -54,7 +54,7 @@ def __init__(self, self._server = None # type: Optional[AbstractServer] self._server_task = None # type: Optional[Task] self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] - self._read_tasks = set() + self._read_tasks = set() # type: Set[Task] async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: """ @@ -84,6 +84,8 @@ def setup(self): def teardown(self): """Tear the connection down.""" + self.out_queue.put(None) + self._fetch_task.result() for t in self._read_tasks: t.cancel() self._server.close() From e1370b0b211b557c6e9b3b8fefd38146cbe771ba Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Nov 2019 19:12:30 +0100 Subject: [PATCH 35/38] improve tear down procedure. --- aea/connections/tcp/base.py | 44 +++++++++++++++++++------------ aea/connections/tcp/tcp_client.py | 30 ++++++--------------- aea/connections/tcp/tcp_server.py | 25 +++++++++++------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/aea/connections/tcp/base.py b/aea/connections/tcp/base.py index 91617a1325..61ed7c63e6 100644 --- a/aea/connections/tcp/base.py +++ b/aea/connections/tcp/base.py @@ -24,8 +24,8 @@ import struct import threading from abc import ABC, abstractmethod -from asyncio import CancelledError, StreamWriter, StreamReader, AbstractEventLoop, Task -from concurrent.futures import Executor +from asyncio import CancelledError, StreamWriter, StreamReader, AbstractEventLoop, Future +from concurrent.futures import Executor, ThreadPoolExecutor from threading import Thread from typing import Optional @@ -51,13 +51,14 @@ def __init__(self, self.host = host self.port = port self._loop = asyncio.new_event_loop() if loop is None else loop + self._executor = executor if executor is not None else ThreadPoolExecutor() self._lock = threading.Lock() self._stopped = True self._connected = False self._thread_loop = None # type: Optional[Thread] - self._recv_task = None # type: Optional[Task] - self._fetch_task = None # type: Optional[Task] + self._recv_task = None # type: Optional[Future] + self._fetch_task = None # type: Optional[Future] def _run_task(self, coro): return asyncio.run_coroutine_threadsafe(coro=coro, loop=self._loop) @@ -111,17 +112,27 @@ def connect(self): Set up the connection. :return: A queue or None. + :raises ConnectionError: if a problem occurred during the connection. """ with self._lock: - if self.is_established: - logger.warning("Connection already set up.") - return + try: + if self.is_established: + logger.warning("Connection already set up.") + return - self._stopped = False - if not self._is_threaded: - self._start_loop() - self.setup() - self._connected = True + self._stopped = False + if not self._is_threaded: + self._start_loop() + + self.setup() + + self._connected = True + except Exception as e: + logger.error(str(e)) + if not self._is_threaded: + self._stop_loop() + self._connected = False + self._stopped = True def disconnect(self) -> None: """ @@ -155,14 +166,13 @@ async def _recv(self, reader: StreamReader) -> Optional[bytes]: return data except CancelledError: logger.debug("[{}] Read cancelled.".format(self.public_key)) - raise + return None except struct.error as e: logger.debug("Struct error: {}".format(str(e))) + return None except Exception as e: logger.exception(e) raise - finally: - return None async def _send(self, writer: StreamWriter, data: bytes) -> None: """Send bytes.""" @@ -199,7 +209,7 @@ async def _send_loop(self): """Process outgoing messages.""" try: logger.debug("[{}]: Waiting for sending next message...".format(self.public_key)) - envelope = await self._loop.run_in_executor(None, self.out_queue.get, True) + envelope = await self._loop.run_in_executor(self._executor, self.out_queue.get, True) if envelope is None: logger.debug("[{}] Stopped sending loop.".format(self.public_key)) return @@ -222,4 +232,4 @@ def send(self, envelope: Envelope) -> None: :param envelope: the envelope to send. :return: None. """ - self.out_queue.put(envelope) + self.out_queue.put_nowait(envelope) diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py index 1f871a7bcc..7e958f5988 100644 --- a/aea/connections/tcp/tcp_client.py +++ b/aea/connections/tcp/tcp_client.py @@ -21,8 +21,7 @@ import asyncio import logging from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader -from concurrent.futures import CancelledError -from threading import Thread +from concurrent.futures import CancelledError, Executor from typing import Optional, cast from aea.configurations.base import ConnectionConfig @@ -42,7 +41,8 @@ def __init__(self, public_key: str, host: str, port: int, - loop: Optional[AbstractEventLoop] = None): + loop: Optional[AbstractEventLoop] = None, + executor: Optional[Executor] = None): """ Initialize a TCP channel. @@ -50,41 +50,27 @@ def __init__(self, :param host: the socket bind address. :param loop: the asyncio loop. """ - super().__init__(public_key, host, port, loop=loop) + super().__init__(public_key, host, port, loop=loop, executor=executor) self._reader, self._writer = (None, None) # type: Optional[StreamReader], Optional[StreamWriter] self._read_task = None # type: Optional[Task] - def _run_task(self, coro): - assert self._loop.is_running() - return asyncio.run_coroutine_threadsafe(coro=coro, loop=self._loop) - - def _start_loop(self): - assert self._thread_loop is None - self._thread_loop = Thread(target=self._loop.run_forever) - self._thread_loop.start() - - def _stop_loop(self): - assert self._thread_loop.is_alive() - self._loop.call_soon_threadsafe(self._loop.stop) - self._thread_loop.join(timeout=10) - self._thread_loop = None - def setup(self): """Set the connection up.""" future = self._run_task(asyncio.open_connection(self.host, self.port, loop=self._loop)) self._reader, self._writer = future.result() public_key_bytes = self.public_key.encode("utf-8") future = self._run_task(self._send(self._writer, public_key_bytes)) - future.result() - self._read_task = self._run_task(self._recv_loop(self._reader)) # TODO store future to handle cancellation + future.result(timeout=3.0) + self._read_task = self._run_task(self._recv_loop(self._reader)) self._fetch_task = self._run_task(self._send_loop()) def teardown(self): """Tear the connection down.""" try: - self.out_queue.put(None) + self.out_queue.put_nowait(None) self._fetch_task.result() + self._reader.feed_eof() self._read_task.cancel() except CancelledError: pass diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index 8b9834cc52..2fcb184cf9 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -22,7 +22,8 @@ import asyncio import logging from asyncio import AbstractEventLoop, StreamReader, StreamWriter, Task, AbstractServer -from typing import Dict, Optional, Tuple, cast, Set +from concurrent.futures import Executor +from typing import Dict, Optional, Tuple, cast from aea.configurations.base import ConnectionConfig from aea.connections.base import Connection @@ -41,7 +42,8 @@ def __init__(self, public_key: str, host: str, port: int, - loop: Optional[AbstractEventLoop] = None): + loop: Optional[AbstractEventLoop] = None, + executor: Optional[Executor] = None): """ Initialize a TCP channel. @@ -49,12 +51,12 @@ def __init__(self, :param host: the socket bind address. :param loop: the asyncio loop. """ - super().__init__(public_key, host, port, loop=loop) + super().__init__(public_key, host, port, loop=loop, executor=executor) self._server = None # type: Optional[AbstractServer] self._server_task = None # type: Optional[Task] self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] - self._read_tasks = set() # type: Set[Task] + self._read_tasks = dict() # type: Dict[str, Task] async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: """ @@ -71,23 +73,26 @@ async def handle(self, reader: StreamReader, writer: StreamWriter) -> None: public_key = public_key_bytes.decode("utf-8") logger.debug("Public key of the client: {}".format(public_key)) self.connections[public_key] = (reader, writer) - task = self._loop.create_task(self._recv_loop(reader)) - self._read_tasks.add(task) + task = self._run_task(self._recv_loop(reader)) + self._read_tasks[public_key] = task def setup(self): """Set the connection up.""" - future = self._run_task(asyncio.start_server(self.handle, host=self.host, port=self.port, - loop=self._loop, start_serving=False)) + future = self._run_task(asyncio.start_server(self.handle, host=self.host, port=self.port, loop=self._loop)) self._server = future.result() self._server_task = self._run_task(self._server.serve_forever()) self._fetch_task = self._run_task(self._send_loop()) def teardown(self): """Tear the connection down.""" - self.out_queue.put(None) + self.out_queue.put_nowait(None) self._fetch_task.result() - for t in self._read_tasks: + + for pbk, (reader, _) in self.connections.items(): + reader.feed_eof() + t = self._read_tasks.get(pbk) t.cancel() + self._server.close() def select_writer_from_envelope(self, envelope: Envelope): From 504d3eb5cea4be9bf0c1e64005dad7da501bc1d4 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Nov 2019 17:00:28 +0100 Subject: [PATCH 36/38] make tcp server connection work with Python 3.6. --- aea/connections/tcp/tcp_client.py | 4 ++-- aea/connections/tcp/tcp_server.py | 3 +-- tests/test_connections/test_tcp/test_communication.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/aea/connections/tcp/tcp_client.py b/aea/connections/tcp/tcp_client.py index 7e958f5988..f492bc5540 100644 --- a/aea/connections/tcp/tcp_client.py +++ b/aea/connections/tcp/tcp_client.py @@ -17,7 +17,7 @@ # # ------------------------------------------------------------------------------ -"""Implementation of the TCP server.""" +"""Implementation of the TCP client.""" import asyncio import logging from asyncio import AbstractEventLoop, Task, StreamWriter, StreamReader @@ -35,7 +35,7 @@ class TCPClientConnection(TCPConnection): - """Abstract TCP channel.""" + """This class implements a TCP client.""" def __init__(self, public_key: str, diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index 2fcb184cf9..1bab520f15 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -36,7 +36,7 @@ class TCPServerConnection(TCPConnection): - """Abstract TCP channel.""" + """This class implements a TCP server.""" def __init__(self, public_key: str, @@ -80,7 +80,6 @@ def setup(self): """Set the connection up.""" future = self._run_task(asyncio.start_server(self.handle, host=self.host, port=self.port, loop=self._loop)) self._server = future.result() - self._server_task = self._run_task(self._server.serve_forever()) self._fetch_task = self._run_task(self._send_loop()) def teardown(self): diff --git a/tests/test_connections/test_tcp/test_communication.py b/tests/test_connections/test_tcp/test_communication.py index 459175ca76..d1bc922883 100644 --- a/tests/test_connections/test_tcp/test_communication.py +++ b/tests/test_connections/test_tcp/test_communication.py @@ -31,7 +31,6 @@ class TestTCPCommunication: @classmethod def setup_class(cls): """Set up the test class.""" - cls.host = "127.0.0.1" cls.port = 8082 From cfe30cb2c71bbadeaa7b3d79e0c4bfd96ed44ec6 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Nov 2019 17:01:40 +0100 Subject: [PATCH 37/38] remove unused attribute. --- aea/connections/tcp/tcp_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aea/connections/tcp/tcp_server.py b/aea/connections/tcp/tcp_server.py index 1bab520f15..959dfa8679 100644 --- a/aea/connections/tcp/tcp_server.py +++ b/aea/connections/tcp/tcp_server.py @@ -54,7 +54,6 @@ def __init__(self, super().__init__(public_key, host, port, loop=loop, executor=executor) self._server = None # type: Optional[AbstractServer] - self._server_task = None # type: Optional[Task] self.connections = {} # type: Dict[str, Tuple[StreamReader, StreamWriter]] self._read_tasks = dict() # type: Dict[str, Task] From b1e4403095e9bc790c121cac7e07266a125d5773 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sun, 3 Nov 2019 16:19:39 +0000 Subject: [PATCH 38/38] Update HISTORY.rst --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index f989a05514..923ce5b2ea 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -95,6 +95,7 @@ Release History 0.1.12 (2019-11-01) ------------------- +- Adds TCP connection (server and client) - Fixes some examples and docs - Refactors crypto modules and adds additional tests - Multiple additional minor fixes and changes