Fix race in test 408 for rabbitmq

There is a race in the 408 test for rabbitmq where the config-change to
enable ssl causes a leader-settings-changed hook in the non-leader units
which results in a rabbitmq service restart.  This can happen at exactly
the same time as the test attempts to establish a connection with the
that unit. This patch retries the connection attempt.

Note that this may only be a partial fix as it's possible that a restart
will happen just after the connection is made, which would then result
in a test failure.

Related-Bug: LP#2002156

tox.ini

@@ -5,51 +5,19 @@ skipsdist = True
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.2.0
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
-install_command =
-  {toxinidir}/pip.sh install {opts} {packages}
-
 commands = pytest --cov=zaza.openstack {posargs} {toxinidir}/unit_tests
-
-[testenv:py3]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.8]
-basepython = python3.8
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.9]
-basepython = python3.9
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.10]
-basepython = python3.10
 deps = -r{toxinidir}/requirements.txt
 
 [testenv:pep8]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
 commands = flake8 {posargs} zaza unit_tests
 
 [testenv:venv]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
 commands = /bin/true
 
 [flake8]
@@ -58,8 +26,5 @@ per-file-ignores =
   unit_tests/**: D
 
 [testenv:docs]
-basepython = python3
 changedir = doc/source
-deps =
-    -r{toxinidir}/requirements.txt
 commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html

zaza/openstack/charm_tests/rabbitmq_server/utils.py

@@ -427,17 +427,25 @@ def connect_amqp_by_unit(unit, ssl=False,
                   '{}...'.format(host, port, unit_name, username))
 
     try:
-        credentials = pika.PlainCredentials(username, password)
-        parameters = pika.ConnectionParameters(host=host, port=port,
-                                               credentials=credentials,
-                                               ssl_options=ssl_options,
-                                               connection_attempts=3,
-                                               retry_delay=5,
-                                               socket_timeout=1)
-        connection = pika.BlockingConnection(parameters)
-        assert connection.is_open is True
-        logging.debug('Connect OK')
-        return connection
+        # retry connections; it's possible during the testing that a
+        # leader-setting-change hook will be running on the unit (which takes
+        # up to 30s to run) and results in a restart of the underlying rabbitmq
+        # process.  This retry get's past the restart.
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(5),
+                wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)):
+            with attempt:
+                credentials = pika.PlainCredentials(username, password)
+                parameters = pika.ConnectionParameters(host=host, port=port,
+                                                       credentials=credentials,
+                                                       ssl_options=ssl_options,
+                                                       connection_attempts=3,
+                                                       retry_delay=5,
+                                                       socket_timeout=1)
+                connection = pika.BlockingConnection(parameters)
+                assert connection.is_open is True
+                logging.debug('Connect OK')
+                return connection
     except Exception as e:
         msg = ('amqp connection failed to {}:{} as '
                '{} ({})'.format(host, port, username, str(e)))