Merge pull request #1167 from dimaqq/fix-matrix-fail

#1167

Looks like I was wrong in #1163 
Using https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast instead.
Skip tests that are guaranteed to fail against older Juju controllers.

.github/workflows/test.yaml

@@ -97,6 +97,7 @@ jobs:
     timeout-minutes: 150
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python:
           # We will reduce the workload to 3.10 to
@@ -114,7 +115,6 @@ jobs:
           # * test_ssh
           # * ...
           # - "3.6/beta"
-    continue-on-error: false  # ultimately fail a run if one of the matrix combinations fails
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -158,14 +158,14 @@ jobs:
       - name: Run integration
         # Force one single concurrent test
         run: tox -e integration
-        continue-on-error: true  # don't fail early, let other matrix combinations get tested
 
   integration-quarantine:
     name: Quarantined Integration Tests
     needs: [lint, unit-tests]
     timeout-minutes: 150
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python:
           - "3.10"
@@ -174,10 +174,9 @@ jobs:
           - "3.3/stable"
           - "3.4/stable"
           - "3.5/stable"
-    continue-on-error: false  # ultimately fail the run if one of the matrix combinations fails
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
@@ -189,4 +188,3 @@ jobs:
           juju-channel: ${{ matrix.juju }}
       - name: Run integration
         run: tox -e integration-quarantine
-        continue-on-error: true  # don't fail early, let other matrix combinations get tested

tests/integration/test_crossmodel.py

@@ -98,6 +98,10 @@ async def test_remove_saas():
 async def test_relate_with_offer():
     # pytest.skip('Revise: intermittent problem with the remove_saas call')
     async with base.CleanModel() as model_1:
+        assert model_1._info
+        if str(model_1._info.agent_version) < "3.4.3":
+            pytest.skip("postgresql charm requires Juju 3.4.3 or later")
+
         application = await model_1.deploy(
             'postgresql',
             application_name='postgresql',

tests/integration/test_model.py

@@ -217,6 +217,10 @@ async def test_deploy_bundle_with_storage_constraint():
     bundle_path = INTEGRATION_TEST_DIR / 'bundle' / 'bundle-with-storage-constraint.yaml'
 
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("bundle/postgresql charm requires Juju 3.4.3 or later")
+
         await model.deploy(bundle_path)
         await wait_for_bundle(model, bundle_path)
         storage = await model.list_storage()
@@ -237,6 +241,10 @@ async def test_deploy_local_charm():
 @base.bootstrapped
 async def test_deploy_charm_assumes():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("postgresql charm requires Juju 3.4.3 or later")
+
         await model.deploy('postgresql', channel='14/edge')
 
 
@@ -298,6 +306,10 @@ async def test_deploy_bundle():
 @pytest.mark.bundle
 async def test_deploy_local_bundle_with_overlay_multi():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("bundle/postgresql charm requires Juju 3.4.3 or later")
+
         bundle_with_overlay_path = OVERLAYS_DIR / 'bundle-with-overlay-multi.yaml'
         await model.deploy(bundle_with_overlay_path)
 
@@ -312,6 +324,10 @@ async def test_deploy_local_bundle_with_overlay_multi():
 @pytest.mark.skip("Always fails -- investigate bundle charms")
 async def test_deploy_bundle_with_overlay_as_argument():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("bundle/postgresql charm requires Juju 3.4.3 or later")
+
         overlay_path = OVERLAYS_DIR / 'test-overlay.yaml'
 
         await model.deploy('juju-qa-bundle-test', overlays=[overlay_path])
@@ -333,6 +349,10 @@ async def test_deploy_bundle_with_overlay_as_argument():
 @pytest.mark.bundle
 async def test_deploy_bundle_with_multi_overlay_as_argument():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("bundle/postgresql charm requires Juju 3.4.3 or later")
+
         overlay_path = OVERLAYS_DIR / 'test-multi-overlay.yaml'
 
         await model.deploy('juju-qa-bundle-test', overlays=[overlay_path])
@@ -381,6 +401,10 @@ async def test_deploy_local_charm_folder_symlink():
 @base.bootstrapped
 async def test_deploy_from_ch_channel_revision_success():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.4.3":
+            pytest.skip("postgresql charm requires Juju 3.4.3 or later")
+
         # Ensure we're able to resolve charm these with channel and revision,
         # or channel without revision (note that revision requires channel,
         # but not vice versa)

tests/integration/test_secrets.py

@@ -10,6 +10,10 @@
 @pytest.mark.bundle
 async def test_add_secret():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         secret = await model.add_secret(name='my-apitoken', data_args=['token=34ae35facd4'])
         assert secret.startswith('secret:')
 
@@ -28,6 +32,10 @@ async def test_list_secrets():
     charm_path = TESTS_DIR / 'charm-secret/charm-secret_ubuntu-22.04-amd64.charm'
 
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         await model.deploy(str(charm_path))
         assert 'charm-secret' in model.applications
         await model.wait_for_idle(status="active")
@@ -42,6 +50,10 @@ async def test_list_secrets():
 @pytest.mark.bundle
 async def test_update_secret():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         secret = await model.add_secret(name='my-apitoken', data_args=['token=34ae35facd4'])
         assert secret.startswith('secret:')
 
@@ -56,6 +68,10 @@ async def test_update_secret():
 @pytest.mark.bundle
 async def test_remove_secret():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         secret = await model.add_secret(name='my-apitoken', data_args=['token=34ae35facd4'])
         assert secret.startswith('secret:')
 
@@ -69,6 +85,10 @@ async def test_remove_secret():
 @pytest.mark.bundle
 async def test_grant_secret():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         secret = await model.add_secret(name='my-apitoken', data_args=['token=34ae35facd4'])
         assert secret.startswith('secret:')
 
@@ -81,6 +101,10 @@ async def test_grant_secret():
 @pytest.mark.bundle
 async def test_revoke_secret():
     async with base.CleanModel() as model:
+        assert model._info
+        if str(model._info.agent_version) < "3.3.0":
+            pytest.skip("Juju too old, need Secrets API v2")
+
         secret = await model.add_secret(name='my-apitoken', data_args=['token=34ae35facd4'])
         assert secret.startswith('secret:')
         await model.revoke_secret('my-apitoken', 'ubuntu')