Merge pull request #699 from communitiesuk/bau/playwright-dev-test

Update e2e tests to work on dev and test envs

.github/workflows/test_e2e.yml

@@ -0,0 +1,46 @@
+name: Run e2e (browser) tests
+on:
+  workflow_call:
+    inputs:
+      copilot_environment:
+        description: "Copilot environment to deploy to"
+        type: string
+        default: dev
+  workflow_dispatch:
+    inputs:
+      copilot_environment:
+        description: "Copilot environment to deploy to"
+        type: choice
+        options:
+          - dev
+          - test
+        default: dev
+
+jobs:
+  run_tests:
+    name: e2e tests
+    permissions:
+      id-token: write # This is required for requesting the JWT
+      contents: read  # This is required for actions/checkout
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.copilot_environment }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip' # caching pip dependencies
+    - name: Install dependencies
+      run: pip install -r requirements-dev.txt
+    - name: Install playwright browsers
+      run: playwright install --with-deps
+    - name: Setup AWS credentials
+      uses: communitiesuk/funding-service-design-workflows/.github/actions/copilot_setup@main
+      with:
+        copilot_environment: ${{ inputs.copilot_environment }}
+        AWS_ACCOUNT: ${{ secrets.AWS_ACCOUNT }}
+    - name: Run tests
+      run: pytest tests/e2e_tests --e2e --e2e-env ${{ inputs.copilot_environment }}
+      env:
+        E2E_DEVTEST_BASIC_AUTH_USERNAME: ${{ secrets.E2E_DEVTEST_BASIC_AUTH_USERNAME }}
+        E2E_DEVTEST_BASIC_AUTH_PASSWORD: ${{ secrets.E2E_DEVTEST_BASIC_AUTH_USERNAME }}

README.md

@@ -103,11 +103,19 @@ e. you won't see a browser appear. To display the browser so you can visually in
 --e2e --headed --slowmo 1000`. `--headed` displays the browser, and `--slowmo 1000` makes Playwright insert 1 second
 pauses between various steps so that you can follow what the test is doing more easily.
 
-The e2e test for find currently requires a `test`-scoped API key for GOV.UK Notify to retrieve the email send during
-the test.  Pass an environment variable called `E2E_NOTIFY_FIND_API_KEY` to allow this test to pass.
+#### Against your local docker-compose services
 
-If you want to run the end-to-end tests against our deployed dev/test environments, you can do this by adding the
-`--e2e-env` flag to the pytest command with a value of either `dev` or `test`.
+Two environment variables must be set:
+`E2E_NOTIFY_FIND_API_KEY` and `E2E_NOTIFY_SUBMIT_API_KEY`. These should be the same as the values used by docker-compose.
+
+#### Against deployed dev/test environments
+
+Two additional flags must be passed to the `pytest` command:
+
+* `--e2e-env` flag to the pytest command with a value of either `dev` or `test`
+* `--e2e-aws-vault-profile` with a value that matches the aws-vault profile name for the matching environment. The
+  tests expect a session to be available without any input, so you must have authenticated already and have your
+  credentials cached.
 
 ## Updating database migrations
 

config/envs/unit_test.py

@@ -1,5 +1,3 @@
-import os
-
 from botocore.config import Config
 from fsd_utils import configclass
 
@@ -59,7 +57,3 @@ class UnitTestConfig(DefaultConfig):
     # Which is overkill for now. 28/06/2024.
     CELERY = DefaultConfig.CELERY
     CELERY["task_always_eager"] = True
-
-    E2E_NOTIFY_API_KEY = os.environ.get("E2E_NOTIFY_API_KEY")
-    E2E_NOTIFY_FIND_API_KEY = os.environ.get("E2E_NOTIFY_FIND_API_KEY")
-    E2E_DEVTEST_BASIC_AUTH = os.environ.get("E2E_DEVTEST_BASIC_AUTH")  # Format: "username:password"

tests/conftest.py

@@ -54,13 +54,23 @@
 
 def pytest_addoption(parser):
     parser.addoption("--e2e", action="store_true", default=False, help="run e2e (browser) tests")
+
+    # WARNING: Do not add an option for `prod` here. We *must* rework the e2e test authentication process before
+    #          that would be something we could consider.
     parser.addoption(
         "--e2e-env",
         action="store",
         default="local",
         help="choose the environment that e2e tests will target",
         choices=("local", "dev", "test"),
     )
+
+    parser.addoption(
+        "--e2e-aws-vault-profile",
+        action="store",
+        help="the aws-vault profile matching the env set in --e2e-env (for `dev` or `test` only)",
+    )
+
     parser.addoption(
         "--viewport",
         default="1920x1080",

tests/e2e_tests/config.py

@@ -0,0 +1,111 @@
+import base64
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import Literal, Protocol
+
+import boto3
+from playwright.sync_api import HttpCredentials
+
+
+class EndToEndTestSecrets(Protocol):
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials | None: ...
+
+    @property
+    def JWT_SIGNING_KEY(self) -> str: ...
+
+    @property
+    def NOTIFY_FIND_API_KEY(self) -> str: ...
+
+    @property
+    def NOTIFY_SUBMIT_API_KEY(self) -> str: ...
+
+
+class LocalEndToEndSecrets:
+    @property
+    def HTTP_BASIC_AUTH(self) -> None:
+        return None
+
+    @property
+    def JWT_SIGNING_KEY(self) -> str:
+        _test_private_key_path = str(Path(__file__).parent.parent) + "/keys/rsa256/private.pem"
+        with open(_test_private_key_path, mode="r") as private_key_file:
+            rsa256_private_key = private_key_file.read()
+
+        return rsa256_private_key
+
+    @property
+    def NOTIFY_FIND_API_KEY(self) -> str:
+        return os.environ["E2E_NOTIFY_FIND_API_KEY"]
+
+    @property
+    def NOTIFY_SUBMIT_API_KEY(self) -> str:
+        return os.environ["E2E_NOTIFY_SUBMIT_API_KEY"]
+
+
+class AWSEndToEndSecrets:
+    def __init__(self, e2e_env: Literal["dev", "test"], e2e_aws_vault_profile: str | None):
+        self.e2e_env = e2e_env
+        self.e2e_aws_vault_profile = e2e_aws_vault_profile
+
+        if self.e2e_env == "prod":  # type: ignore[comparison-overlap]
+            raise ValueError("shouldn't be possible, but also must never happen")
+
+    def _read_aws_parameter_store_value(self, parameter):
+        # This flow is used to collect secrets when running tests *from* your local machine
+        if self.e2e_aws_vault_profile:
+            value = json.loads(
+                subprocess.check_output(
+                    [
+                        "aws-vault",
+                        "exec",
+                        self.e2e_aws_vault_profile,
+                        "--",
+                        "aws",
+                        "ssm",
+                        "get-parameter",
+                        "--name",
+                        parameter,
+                        "--with-decryption",
+                    ],
+                ).decode()
+            )["Parameter"]["Value"]
+
+        # This flow is used when running tests *in* CI/CD, where AWS credentials are available from OIDC auth
+        else:
+            ssm_client = boto3.client("ssm")
+            value = ssm_client.get_parameter(Name=parameter, WithDecryption=True)["Parameter"]["Value"]
+
+        return value
+
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials:
+        return {
+            "username": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/POST_AWARD_BASIC_AUTH_USERNAME"
+            ),
+            "password": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/POST_AWARD_BASIC_AUTH_PASSWORD"
+            ),
+        }
+
+    @property
+    def JWT_SIGNING_KEY(self) -> str:
+        base64_value = self._read_aws_parameter_store_value(
+            f"/copilot/pre-award/{self.e2e_env}/secrets/RSA256_PRIVATE_KEY_BASE64"
+        )
+        return base64.b64decode(base64_value).decode()
+
+    @property
+    def NOTIFY_FIND_API_KEY(self) -> str:
+        return self._read_aws_parameter_store_value(
+            f"/copilot/pre-award/{self.e2e_env}/secrets/POST_AWARD_NOTIFY_FIND_API_KEY"
+        )
+
+    @property
+    def NOTIFY_SUBMIT_API_KEY(self) -> str:
+        return self._read_aws_parameter_store_value(
+            f"/copilot/pre-award/{self.e2e_env}/secrets/POST_AWARD_NOTIFY_API_KEY"
+        )

tests/e2e_tests/conftest.py

@@ -1,13 +1,15 @@
+import datetime
+import uuid
+
+import jwt
 import pytest
-import requests
-from playwright._impl._errors import Error as PlaywrightError
-from playwright.sync_api import Page
+from playwright.sync_api import Browser, BrowserContext, Page
 from pytest import FixtureRequest
 
 from config import Config
+from tests.e2e_tests.config import AWSEndToEndSecrets, EndToEndTestSecrets, LocalEndToEndSecrets
 from tests.e2e_tests.dataclasses import Account, FundingServiceDomains, TestFundConfig
-from tests.e2e_tests.helpers import create_account_with_roles, generate_email_address
-from tests.e2e_tests.pages.authenticator import MagicLinkPage, NewMagicLinkPage
+from tests.e2e_tests.helpers import generate_email_address
 
 
 @pytest.fixture(autouse=True)
@@ -19,28 +21,27 @@ def _viewport(request: FixtureRequest, page: Page):
 @pytest.fixture()
 def domains(request: FixtureRequest) -> FundingServiceDomains:
     e2e_env = request.config.getoption("e2e_env")
-    devtest_basic_auth = Config.E2E_DEVTEST_BASIC_AUTH
-
-    if e2e_env in {"dev", "test"} and not devtest_basic_auth:
-        raise ValueError("E2E_DEVTEST_BASIC_AUTH is not set to `username:password` for accessing dev/test environments")
 
     if e2e_env == "local":
         return FundingServiceDomains(
+            cookie=".levellingup.gov.localhost",
             authenticator=f"http://{Config.AUTHENTICATOR_HOST}.levellingup.gov.localhost:4004",
             find=f"http://{Config.FIND_HOST}",
             submit=f"http://{Config.SUBMIT_HOST}",
         )
     elif e2e_env == "dev":
         return FundingServiceDomains(
-            authenticator=f"https://{devtest_basic_auth}@authenticator.dev.access-funding.test.levellingup.gov.uk",
-            find=f"https://{devtest_basic_auth}@find-monitoring-data.dev.access-funding.test.levellingup.gov.uk",
-            submit=f"https://{devtest_basic_auth}@submit-monitoring-data.dev.access-funding.test.levellingup.gov.uk",
+            cookie=".dev.access-funding.test.levellingup.gov.uk",
+            authenticator="https://authenticator.dev.access-funding.test.levellingup.gov.uk",
+            find="https://find-monitoring-data.dev.access-funding.test.levellingup.gov.uk",
+            submit="https://submit-monitoring-data.dev.access-funding.test.levellingup.gov.uk",
         )
     elif e2e_env == "test":
         return FundingServiceDomains(
-            authenticator=f"https://{devtest_basic_auth}@authenticator.test.access-funding.test.levellingup.gov.uk",
-            find=f"https://{devtest_basic_auth}@find-monitoring-data.test.access-funding.test.levellingup.gov.uk",
-            submit=f"https://{devtest_basic_auth}@submit-monitoring-data.test.access-funding.test.levellingup.gov.uk",
+            cookie=".test.access-funding.test.levellingup.gov.uk",
+            authenticator="https://authenticator.test.access-funding.test.levellingup.gov.uk",
+            find="https://find-monitoring-data.test.access-funding.test.levellingup.gov.uk",
+            submit="https://submit-monitoring-data.test.access-funding.test.levellingup.gov.uk",
         )
     else:
         raise ValueError(f"not configured for {e2e_env}")
@@ -65,50 +66,68 @@ def authenticator_fund_config(request: FixtureRequest) -> TestFundConfig:
         raise ValueError(f"not configured for {e2e_env}")
 
 
+@pytest.fixture
+def context(request: FixtureRequest, browser: Browser, e2e_test_secrets: EndToEndTestSecrets):
+    e2e_env = request.config.getoption("e2e_env")
+    http_credentials = e2e_test_secrets.HTTP_BASIC_AUTH if e2e_env in {"dev", "test"} else None
+    return browser.new_context(http_credentials=http_credentials)
+
+
+@pytest.fixture
+def e2e_test_secrets(request: FixtureRequest) -> EndToEndTestSecrets:
+    e2e_env = request.config.getoption("e2e_env")
+    e2e_aws_vault_profile = request.config.getoption("e2e_aws_vault_profile")
+
+    if e2e_env == "local":
+        return LocalEndToEndSecrets()
+
+    if e2e_env in {"dev", "test"}:
+        return AWSEndToEndSecrets(e2e_env=e2e_env, e2e_aws_vault_profile=e2e_aws_vault_profile)
+
+    raise ValueError(f"Unknown e2e_env: {e2e_env}.")
+
+
 @pytest.fixture()
 def user_auth(
     request: FixtureRequest,
     domains: FundingServiceDomains,
     authenticator_fund_config: TestFundConfig,
-    page: Page,
+    context: BrowserContext,
+    e2e_test_secrets: EndToEndTestSecrets,
 ) -> Account:
     email_address = generate_email_address(
         test_name=request.node.originalname,
         email_domain="communities.gov.uk",
     )
 
-    user_roles = request.node.get_closest_marker("user_roles")
-
-    account = create_account_with_roles(
-        email_address=email_address,
-        roles=user_roles.args[0] if user_roles else [],
+    roles_marker = request.node.get_closest_marker("user_roles")
+    user_roles = roles_marker.args[0] if roles_marker else []
+
+    now = int(datetime.datetime.timestamp(datetime.datetime.now()))
+    jwt_data = {
+        "accountId": str(uuid.uuid4()),
+        "azureAdSubjectId": str(uuid.uuid4()),
+        "email": email_address,
+        "fullName": f"E2E Test User - {request.node.originalname}",
+        "roles": user_roles,
+        "iat": now,
+        "exp": now + (15 * 60),  # 15 minutes from now
+    }
+
+    # Algorithm below must match that used by fsd-authenticator
+    cookie_value = jwt.encode(jwt_data, e2e_test_secrets.JWT_SIGNING_KEY, algorithm="RS256")
+
+    context.add_cookies(
+        [
+            {
+                "name": Config.FSD_USER_TOKEN_COOKIE_NAME,
+                "value": cookie_value,
+                "domain": domains.cookie,
+                "path": "/",
+                "httpOnly": True,
+                "secure": True,
+            }
+        ]
     )
 
-    response = requests.get(f"{domains.authenticator}/magic-links")
-    magic_links_before = set(response.json())
-
-    new_magic_link_page = NewMagicLinkPage(page, domain=domains.authenticator, fund_config=authenticator_fund_config)
-    new_magic_link_page.navigate()
-    new_magic_link_page.insert_email_address(account.email_address)
-    new_magic_link_page.press_continue()
-
-    response = requests.get(f"{domains.authenticator}/magic-links")
-    magic_links_after = set(response.json())
-
-    new_magic_links = magic_links_after - magic_links_before
-    for magic_link in new_magic_links:
-        if magic_link.startswith("link:"):
-            break
-    else:
-        raise KeyError("Could not generate/retrieve a new magic link via authenticator")
-
-    magic_link_id = magic_link.split(":")[1]
-    magic_link_page = MagicLinkPage(page, domain=domains.authenticator)
-
-    try:
-        magic_link_page.navigate(magic_link_id)
-    except PlaywrightError:
-        # FIXME: Authenticator gets into a weird redirect loop locally... We just ignore that error.
-        pass
-
-    return account
+    return Account(email_address=email_address, roles=user_roles)

tests/e2e_tests/dataclasses.py

@@ -9,6 +9,7 @@ class Account:
 
 @dataclass
 class FundingServiceDomains:
+    cookie: str
     authenticator: str
     find: str
     submit: str