Merge pull request #604 from coreycb/antelope-overlay-ppas

Antelope overlay ppas

This also includes several cherry-picks to get the gate passing for this branch.

Cherry-picks included are:

Add overlay_ppa to tests_options
(cherry picked from commit 0c07920)
Updates to overlay_ppa
(cherry picked from commit 9cc938a)
Fix NoneType error in enumerate(overlay_ppas)
(cherry picked from commit 986474b)
Drop py36, py37, and py39 from github runners
(cherry picked from commit 6122daa)
Unpin pip and virtualenv
(cherry picked from commit 55c3f3f)
Bump up flake8
(cherry picked from commit a9d6812)
Migrate from nosetest to pytest
(cherry picked from commit 4b0d23e)

.github/workflows/tox.yaml

@@ -5,36 +5,11 @@ on:
   - pull_request
 
 jobs:
-  build_old_versions:
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.6']
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install codecov tox tox-gh-actions
-    - name: Lint with tox
-      run: tox -e pep8
-    - name: Test with tox
-      run: tox -e py
-    - name: Codecov
-      run: |
-        set -euxo pipefail
-        codecov --verbose --gcov-glob unit_tests/*
-
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.8', '3.10']
 
     steps:
     - uses: actions/checkout@v1

test-requirements.txt

@@ -1,14 +1,13 @@
 aiounittest
-flake8>=2.2.4
+flake8>=5  # Python 3.8 compatibility in pyflakes 2.1.0+
 flake8-docstrings
 flake8-per-file-ignores
 pydocstyle<4.0.0
 coverage
 mock>=1.2
-# For some reason the PyPi distributed wheel of nose differ from the one on
-# GitHub, and it does not work with Python 3.10.
-nose>=1.3.7;python_version<'3.10'
-git+https://github.com/nose-devs/nose.git@release_1.3.7#egg=nose;python_version=='3.10'
+pytest
+pytest-cov
+pytest-asyncio
 
 # TODO: these requirements should be mocked out in unit_tests/__init__.py
 async_generator

tox.ini

@@ -2,28 +2,24 @@
 envlist = pep8,py3
 skipsdist = True
 
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
 # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
 minversion = 3.2.0
 
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
-         PYTHONHASHSEED=0
-whitelist_external = juju
-passenv = HOME TERM CS_* OS_* TEST_*
-deps = -r{toxinidir}/test-requirements.txt
-install_command =
-  {toxinidir}/pip.sh install {opts} {packages}
-commands = nosetests --with-coverage --processes=0 --cover-package=zaza {posargs} {toxinidir}/unit_tests
+setenv =
+  VIRTUAL_ENV={envdir}
+  PYTHONHASHSEED=0
+allowlist_external =
+  juju
+passenv =
+  HOME
+  TERM
+  CS_*
+  OS_*
+  TEST_*
+deps =
+  -r{toxinidir}/test-requirements.txt
+commands = pytest --cov=./zaza/ {posargs} {toxinidir}/unit_tests
 
 [testenv:py3]
 basepython = python3

unit_tests/test_zaza_model.py

@@ -32,6 +32,7 @@
 import concurrent
 import datetime
 import mock
+import pytest
 import yaml
 
 import unit_tests.utils as ut_utils
@@ -100,6 +101,7 @@ def tearDownModule():
                                 }}}}}}
 
 
+@pytest.mark.asyncio
 class TestModel(ut_utils.BaseTestCase):
 
     def setUp(self):

unit_tests/utilities/test_deployment_env.py

@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import copy
 import mock
 import yaml
 
 import zaza.utilities.deployment_env as deployment_env
+import zaza.utilities.ro_types as ro_types
 import unit_tests.utils as ut_utils
 
 
@@ -52,40 +54,98 @@ def test_parse_option_list_string_whitespace(self):
                 'test-mode': 'false',
                 'image-stream': 'released'})
 
+    def test_get_overlay_ppas(self):
+        with mock.patch('zaza.global_options.get_options') as get_options_mock:
+            config = collections.OrderedDict({'overlay_ppas':
+                                              ['ppa:ppa1', 'ppa:ppa2']})
+            get_options_mock.return_value = ro_types.resolve_immutable(config)
+            self.assertEqual(deployment_env.get_overlay_ppas(),
+                             ro_types.ReadOnlyList(['ppa:ppa1', 'ppa:ppa2']))
+
+            config = collections.OrderedDict({'force_deploy': 'x-y'})
+            get_options_mock.return_value = ro_types.resolve_immutable(config)
+            self.assertEqual(deployment_env.get_overlay_ppas(), None)
+
+    def test_get_cloudinit_userdata(self):
+        with mock.patch.object(deployment_env, 'get_overlay_ppas',
+                               return_value=['ppa:ppa0', 'ppa:ppa1']):
+            cloud_config = {
+                'apt': {
+                    'sources': {
+                        'overlay-ppa-0': {
+                            'source': 'ppa:ppa0'
+                        },
+                        'overlay-ppa-1': {
+                            'source': 'ppa:ppa1'
+                        }
+                    }
+                }
+            }
+            cloudinit_userdata = "#cloud-config\n{}".format(
+                yaml.safe_dump(cloud_config))
+            self.assertEqual(
+                deployment_env.get_cloudinit_userdata(),
+                cloudinit_userdata)
+
     def base_get_model_settings(self, env, expect):
         with mock.patch.dict(deployment_env.os.environ, env):
             self.assertEqual(deployment_env.get_model_settings(), expect)
 
     def test_get_model_settings_no_config(self):
-        self.base_get_model_settings({}, self.MODEL_CONFIG_DEFAULTS)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            self.base_get_model_settings({}, self.MODEL_CONFIG_DEFAULTS)
 
     def test_get_model_settings_multiple_values_override(self):
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'test-mode': 'false'})
-        self.base_get_model_settings(
-            {'TEST_MODEL_SETTINGS': 'test-mode=false'},
-            expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'test-mode': 'false'})
+            self.base_get_model_settings(
+                {'TEST_MODEL_SETTINGS': 'test-mode=false'},
+                expect_config)
 
     def test_get_model_settings_file_override(self):
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'default-series': 'file-setting'})
-        self.patch_object(
-            deployment_env,
-            'get_setup_file_section',
-            return_value={'default-series': 'file-setting'})
-        self.base_get_model_settings({}, expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'default-series': 'file-setting'})
+            self.patch_object(
+                deployment_env,
+                'get_setup_file_section',
+                return_value={'default-series': 'file-setting'})
+            self.base_get_model_settings({}, expect_config)
 
     def test_get_model_settings_file_override_env_override(self):
-        # Check that env variables override defaults and file
-        expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
-        expect_config.update({'default-series': 'env-setting'})
-        self.patch_object(
-            deployment_env,
-            'get_setup_file_section',
-            return_value={'default-series': 'file-setting'})
-        self.base_get_model_settings(
-            {'TEST_MODEL_SETTINGS': 'default-series=env-setting'},
-            expect_config)
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value=None):
+            # Check that env variables override defaults and file
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'default-series': 'env-setting'})
+            self.patch_object(
+                deployment_env,
+                'get_setup_file_section',
+                return_value={'default-series': 'file-setting'})
+            self.base_get_model_settings(
+                {'TEST_MODEL_SETTINGS': 'default-series=env-setting'},
+                expect_config)
+
+    def test_get_model_settings_cloudinit_userdata(self):
+        with mock.patch.object(deployment_env, 'get_cloudinit_userdata',
+                               return_value='x'):
+            expect_config = copy.deepcopy(self.MODEL_CONFIG_DEFAULTS)
+            expect_config.update({'cloudinit-userdata': 'x'})
+            self.base_get_model_settings({}, expect_config)
+
+            with mock.patch.object(deployment_env.logging, 'warn') as warn:
+                with mock.patch.dict(deployment_env.os.environ,
+                                     {'TEST_MODEL_SETTINGS':
+                                      "cloudinit-userdata=y"}):
+                    expect_config.update({'cloudinit-userdata': 'y'})
+                    self.base_get_model_settings({}, expect_config)
+                    warn.assert_called_once_with(
+                        'TEST_MODEL_SETTINGS contains cloudinit-userdata '
+                        'which overrides tests_options overlay_ppas')
 
     def base_get_model_constraints(self, env, expect):
         with mock.patch.dict(deployment_env.os.environ, env):

zaza/utilities/deployment_env.py

@@ -19,6 +19,7 @@
 import functools
 import yaml
 
+import zaza.global_options
 import zaza.model
 
 ZAZA_SETUP_FILE_LOCATIONS = [
@@ -84,6 +85,54 @@ def parse_option_list_string(option_list, delimiter=None):
     return settings
 
 
+def get_overlay_ppas():
+    """Get overlay_ppas from global_config.
+
+    In the config file for the tests, the tests_options.overlay_ppa option
+    may be used to specify one or more PPAs that will be enabled for all
+    units in the model.
+
+    The tests_options section needs to look like:
+
+        tests_options:
+          overlay_ppas:
+            - ppa:ubuntu-security-proposed/ppa
+
+    :returns: List of overlay PPAs
+    :rtype: list[str]
+    """
+    config = zaza.global_options.get_options()
+    try:
+        return config.overlay_ppas
+    except KeyError:
+        pass
+    return None
+
+
+def get_cloudinit_userdata():
+    """Return cloudinit_userdata based on tests_options config.
+
+    :returns: YAML-formatted string of cloudinit_userdata
+    :rtype: str
+    """
+    cloudinit_userdata = None
+    cloud_config = {
+        'apt': {
+            'sources': {
+            }
+        }
+    }
+    overlay_ppas = get_overlay_ppas()
+    if overlay_ppas:
+        for index, overlay_ppa in enumerate(overlay_ppas):
+            cloud_config['apt']['sources']["overlay-ppa-{}".format(index)] = {
+                'source': overlay_ppa
+            }
+        cloudinit_userdata = "#cloud-config\n{}".format(
+            yaml.safe_dump(cloud_config))
+    return cloudinit_userdata
+
+
 def get_model_settings():
     """Return model settings from defaults, config file and env variables.
 
@@ -94,6 +143,12 @@ def get_model_settings():
     model_settings.update(get_setup_file_section(MODEL_SETTINGS_SECTION))
     env_settings = os.environ.get('MODEL_SETTINGS', '')
     test_env_settings = os.environ.get('TEST_MODEL_SETTINGS', '')
+    cloudinit_userdata = get_cloudinit_userdata()
+    if cloudinit_userdata:
+        if 'cloudinit-userdata' in test_env_settings:
+            logging.warn('TEST_MODEL_SETTINGS contains cloudinit-userdata '
+                         'which overrides tests_options overlay_ppas')
+        model_settings.update({'cloudinit-userdata': cloudinit_userdata})
     model_settings.update(
         parse_option_list_string(test_env_settings or env_settings))
     if env_settings: