Merge pull request #2371 from fetchai/develop

Release v0.11.2

.coveragerc

@@ -1,2 +1,13 @@
 [run]
-omit = */.tox/*
+omit =
+    */.tox/*
+    packages/fetchai/contracts/*
+    packages/fetchai/skills/aries_alice/*
+    packages/fetchai/skills/aries_faber/*
+    packages/fetchai/skills/echo/*
+    packages/fetchai/skills/erc1155_client/*
+    packages/fetchai/skills/erc1155_deploy/*
+    packages/fetchai/skills/gym/*
+    packages/fetchai/skills/http_echo/*
+    packages/fetchai/skills/registration_aw1/*
+    plugins/aea-ledger-cosmos/aea_ledger_cosmos/cosmos.py

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '35 1 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/workflow.yml

@@ -25,7 +25,13 @@ jobs:
           sudo apt-get autoclean
           pip install pipenv
       - name: Pipenv lock
-        run: pipenv lock
+        run: |
+          pipenv install --dev --skip-lock --clear
+          pipenv run pip install -e .[all]
+          pipenv run pip install --no-deps file:plugins/aea-ledger-ethereum
+          pipenv run pip install --no-deps file:plugins/aea-ledger-cosmos
+          pipenv run pip install --no-deps file:plugins/aea-ledger-fetchai
+          pipenv lock
       - name: Check plugin consistency
         run: |
           # these two files should not be different;
@@ -178,13 +184,20 @@ jobs:
 
   dependencies_checks:
     continue-on-error: False
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python_version: [3.6]
     timeout-minutes: 10
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@master
       - uses: actions/setup-python@master
         with:
-          python-version: 3.6
+          python-version: ${{ matrix.python_version }}
+      - name: Install tox
+        run : |
+          pip install tox
       - name: Check Pipfile and tox.ini consistency
         run : |
           python ./scripts/check_pipfile_and_toxini.py
@@ -193,7 +206,7 @@ jobs:
           pip install -e .[all]
       - name: Check aea dependenices and imports
         run: |
-          python ./scripts/check_imports_and_dependencies.py
+          tox -e dependencies_check
 
   protolint:
     continue-on-error: False
@@ -380,7 +393,7 @@ jobs:
         tox -e py${{ matrix.python_version }} -- -m 'not integration and not unstable'
     - name: Plugin tests
       run: |
-        tox -e plugins-py${{ matrix.python_version }} -- --cov-append -m 'not integration and not unstable'
+        tox -e plugins-py${{ matrix.python_version }} -- -m 'not integration and not unstable'
 
   platform_checks_sync_aea_loop:
     continue-on-error: True
@@ -466,6 +479,7 @@ jobs:
         sudo apt-get autoremove
         sudo apt-get autoclean
         pip install tox
+        pip install coverage
         # install Protobuf compiler
         wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip
         unzip protoc-3.11.4-linux-x86_64.zip -d protoc
@@ -477,6 +491,9 @@ jobs:
         tox -e py3.7-cov -- --ignore=tests/test_docs --ignore=tests/test_examples --ignore=tests/test_packages/test_contracts --ignore=tests/test_packages/test_skills_integration -m 'not unstable'
         tox -e plugins-py3.7-cov -- --cov-append -m 'not unstable'
       continue-on-error: true
+    - name: Show full coverage report
+      run: |
+        coverage report -m -i
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v1
       with:

.gitignore

@@ -129,3 +129,4 @@ packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node
 !tests/data/dummy_contract/build
 !plugins/aea-ledger-ethereum/tests/data/dummy_contract/build
 !plugins/aea-ledger-cosmos/tests/data/dummy_contract/build
+!plugins/aea-ledger-fetchai/tests/data/dummy_contract/build

.pylintrc

@@ -22,7 +22,7 @@ disable=C0103,C0201,C0301,C0302,C0330,W0105,W0107,W0707,W1202,W1203,R0801
 # R0801: similar lines, # too granular
 
 [IMPORTS]
-ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,vyper,web3,aioprometheus
+ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus
 
 [DESIGN]
 min-public-methods=1

HISTORY.md

@@ -1,5 +1,21 @@
 # Release History
 
+## 0.11.2 (2020-03-17)
+
+- Fixes a package import issue
+- Fixes an issue where `AgentLoop` did not teardown properly under certain conditions
+- Fixes a bug in testing tools
+- Fixes a bug where plugins are not loaded after installation in `MultiAgentManager`
+- Adds unit tests for weather, thermometer and car park skills
+- Fixes a missing dependency in Windows
+- Improves SOEF connections' error handling
+- Fixes bug in ML skills and adds unit tests
+- Adds script to bump plugin versions
+- Adds gas price strategy support in `aea-ledger-ethereum` plugin
+- Adds CLI plugin for IPFS interactions (add/get)
+- Adds support for CLI plugins to framework
+- Multiple additional tests and test stability fixes
+
 ## 0.11.1 (2020-03-06)
 
 - Bumps `aiohttp` to `>=3.7.4` to address a CVE affecting `http_server`, `http_client` and `webhook` connections

Makefile

@@ -47,23 +47,21 @@ lint:
 
 .PHONY: pylint
 pylint:
-	pylint -j4 aea benchmark packages scripts plugins/aea-ledger-fetchai/aea_ledger_fetchai plugins/aea-ledger-ethereum/aea_ledger_ethereum plugins/aea-ledger-cosmos/aea_ledger_cosmos examples/*
+	pylint -j4 aea benchmark packages scripts plugins/aea-ledger-fetchai/aea_ledger_fetchai plugins/aea-ledger-ethereum/aea_ledger_ethereum plugins/aea-ledger-cosmos/aea_ledger_cosmos plugins/aea-cli-ipfs/aea_cli_ipfs examples/*
 
 .PHONY: security
 security:
 	bandit -r aea benchmark examples packages \
         plugins/aea-ledger-fetchai/aea_ledger_fetchai \
         plugins/aea-ledger-ethereum/aea_ledger_ethereum \
-        plugins/aea-ledger-cosmos/aea_ledger_cosmos
-	bandit -s B101 -r tests scripts \
-        plugins/aea-ledger-fetchai/tests \
-        plugins/aea-ledger-ethereum/tests \
-        plugins/aea-ledger-cosmos/tests
+        plugins/aea-ledger-cosmos/aea_ledger_cosmos \
+        plugins/aea-cli-ipfs/aea_cli_ipfs
+	bandit -s B101 -r tests scripts
 	safety check -i 37524 -i 38038 -i 37776 -i 38039 -i 39621
 
 .PHONY: static
 static:
-	mypy aea benchmark examples packages plugins/aea-ledger-fetchai/aea_ledger_fetchai plugins/aea-ledger-ethereum/aea_ledger_ethereum plugins/aea-ledger-cosmos/aea_ledger_cosmos scripts --disallow-untyped-defs
+	mypy aea benchmark examples packages plugins/aea-ledger-fetchai/aea_ledger_fetchai plugins/aea-ledger-ethereum/aea_ledger_ethereum plugins/aea-ledger-cosmos/aea_ledger_cosmos plugins/aea-cli-ipfs/aea_cli_ipfs scripts --disallow-untyped-defs
 	mypy tests
 
 .PHONY: package_checks
@@ -84,17 +82,18 @@ test:
 	pytest -rfE plugins/aea-ledger-fetchai/tests --cov=aea_ledger_fetchai --cov-report=term --cov-report=term-missing --cov-config=.coveragerc
 	pytest -rfE plugins/aea-ledger-ethereum/tests --cov=aea_ledger_ethereum --cov-report=term --cov-report=term-missing --cov-config=.coveragerc
 	pytest -rfE plugins/aea-ledger-cosmos/tests --cov=aea_ledger_cosmos --cov-report=term --cov-report=term-missing --cov-config=.coveragerc
-	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search packages/fetchai/skills/advanced_data_request packages/fetchai/skills/fetch_beacon packages/fetchai/skills/simple_oracle packages/fetchai/skills/simple_oracle_client tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/confirmation_aw2 --cov=packages/fetchai/skills/confirmation_aw3 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov=packages/fetchai/skills/advanced_data_request --cov=packages/fetchai/skills/fetch_beacon --cov=packages/fetchai/skills/simple_oracle --cov=packages/fetchai/skills/simple_oracle_client --cov-config=.coveragerc
+	pytest -rfE plugins/aea-cli-ipfs/tests --cov=aea_cli_ipfs --cov-report=term --cov-report=term-missing --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills tests/ --cov=aea --cov=packages/fetchai/connections --cov=packages/fetchai/contracts --cov=packages/fetchai/protocols --cov=packages/fetchai/skills --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub
 test-sub:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub-p
 test-sub-p:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search packages/fetchai/skills/advanced_data_request packages/fetchai/skills/fetch_beacon packages/fetchai/skills/simple_oracle packages/fetchai/skills/simple_oracle_client tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 
@@ -137,6 +136,7 @@ new_env: clean
 		pipenv run pip install --no-deps file:plugins/aea-ledger-ethereum;\
 		pipenv run pip install --no-deps file:plugins/aea-ledger-cosmos;\
 		pipenv run pip install --no-deps file:plugins/aea-ledger-fetchai;\
+		pipenv run pip install --no-deps file:plugins/aea-cli-ipfs;\
 		echo "Enter virtual environment with all development dependencies now: 'pipenv shell'.";\
 	else\
 		echo "In a virtual environment! Exit first: 'exit'.";\

Pipfile

@@ -37,7 +37,7 @@ mkdocs = "==1.1"
 mkdocs-material = "==4.6.3"
 mkdocs-mermaid-plugin = {git = "https://github.com/pugong/mkdocs-mermaid-plugin.git"}
 mypy = "==0.761"
-numpy = "==1.18.1"
+numpy = ">=1.18.1"
 oef = "==0.8.1"
 openapi-core = "==0.13.2"
 openapi-spec-validator = "==0.2.8"
@@ -57,10 +57,12 @@ pytest-randomly = "==3.4.0"
 pytest-rerunfailures = "==9.0"
 requests = ">=2.22.0"
 safety = "==1.10.3"
+scikit-image = ">=0.17.2"
 sqlalchemy = "==1.3.17"
+temper-py = "==0.0.3"
+tensorflow = "==2.4.0"
 tox = "==3.15.1"
 vulture = "==2.3"
-vyper = "==0.1.0b12"
 isort = "==5.7.0"
 web3 = "==5.12.0"
 yoti = "==2.14.0"

aea/__version__.py

@@ -22,7 +22,7 @@
 __title__ = "aea"
 __description__ = "Autonomous Economic Agent framework"
 __url__ = "https://github.com/fetchai/agents-aea.git"
-__version__ = "0.11.1"
+__version__ = "0.11.2"
 __author__ = "Fetch.AI Limited"
 __license__ = "Apache-2.0"
 __copyright__ = "2019 Fetch.AI Limited"

aea/agent_loop.py

@@ -32,12 +32,7 @@
 from aea.abstract_agent import AbstractAgent
 from aea.configurations.constants import LAUNCH_SUCCEED_MESSAGE
 from aea.exceptions import AEAException
-from aea.helpers.async_utils import (
-    AsyncState,
-    HandlerItemGetter,
-    PeriodicCaller,
-    Runnable,
-)
+from aea.helpers.async_utils import AsyncState, PeriodicCaller, Runnable
 from aea.helpers.exec_timeout import ExecTimeoutThreadGuard, TimeoutException
 from aea.helpers.logging import WithLogger, get_logger
 from aea.mail.base import Envelope, EnvelopeContext
@@ -142,14 +137,15 @@ async def _stop(self) -> None:
         self._teardown()
         self._stop_tasks()
         for t in self._tasks:
-            with suppress(BaseException):
+            with suppress(CancelledError, KeyboardInterrupt):
                 await t
+
         self._state.set(AgentLoopStates.stopped)
         self.logger.debug("agent loop stopped")
 
     async def _gather_tasks(self) -> None:
         """Wait till first task exception."""
-        await asyncio.gather(*self._tasks, loop=self._loop)
+        await asyncio.gather(*self._tasks)
 
     @abstractmethod
     def _set_tasks(self) -> None:  # pragma: nocover
@@ -159,8 +155,6 @@ def _set_tasks(self) -> None:  # pragma: nocover
     def _stop_tasks(self) -> None:
         """Cancel all tasks."""
         for task in self._tasks:
-            if task.done():
-                continue  #  pragma: nocover
             task.cancel()
 
     @abstractmethod
@@ -210,8 +204,8 @@ def __init__(
 
     def _setup(self) -> None:
         """Set up agent loop before started."""
-        self._skill2skill_message_queue = asyncio.Queue()
         super()._setup()
+        self._skill2skill_message_queue = asyncio.Queue()
 
     @property
     def skill2skill_queue(self) -> Queue:
@@ -380,24 +374,41 @@ def _create_tasks(self) -> List[Task]:
 
         :return: list of asyncio Tasks
         """
-        tasks = [
-            self._process_messages(HandlerItemGetter(self._message_handlers())),
+        coros = [
+            self._process_messages(),
             self._task_register_periodic_tasks(),
             self._task_wait_for_error(),
         ]
-        return list(map(self._loop.create_task, tasks))  # type: ignore  # some issue with map and create_task
+        return list(map(self._loop.create_task, coros))  # type: ignore  # some issue with map and create_task
+
+    async def _message_processor(
+        self, message_handler: Callable, message_getter: Callable
+    ) -> None:
+        """Fetch messages from the message getter and process it with message handler."""
+        try:
+            while self.is_running:
+                message = await message_getter()
+                self._execution_control(message_handler, [message])
+        except Exception:
+            self.logger.exception(
+                f"Exception in message processor ({message_handler, message_getter})"
+            )
+            raise
 
     def _message_handlers(self) -> List[Tuple[Callable[[Any], None], Callable]]:
         """Get all agent's message handlers."""
         return self._agent.get_message_handlers()
 
-    async def _process_messages(self, getter: HandlerItemGetter) -> None:
-        """Process message from ItemGetter."""
+    async def _process_messages(self) -> None:
+        """Start tasks for messages handlers and sources."""
+        coros = []
+        for handler, getter in self._message_handlers():
+            coros.append(self._message_processor(handler, getter))
+
         self.logger.info(LAUNCH_SUCCEED_MESSAGE)
         self._state.set(AgentLoopStates.started)
-        while self.is_running:
-            handler, item = await getter.get()
-            self._execution_control(handler, [item])
+
+        await asyncio.gather(*coros)
 
     async def _task_register_periodic_tasks(self) -> None:
         """Process new behaviours added to skills in runtime."""