diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 5092900..da23d01 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -21,16 +21,16 @@ from quart import Quart -import trafficlight import trafficlight.kiwi as kiwi from trafficlight.homerunner import HomerunnerClient from trafficlight.http.adapter import ( adapter_shutdown, + loop_check_all_tests_done, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, ) from trafficlight.internals.testsuite import TestSuite -from trafficlight.store import add_testsuite +from trafficlight.store import add_testsuite, get_testsuites from trafficlight.tests import load_tests logger = logging.getLogger(__name__) @@ -114,14 +114,38 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: async def startup() -> None: app.add_background_task(loop_cleanup_unresponsive_adapters) app.add_background_task(loop_check_for_new_tests) + app.add_background_task(loop_check_all_tests_done) if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() @app.after_serving async def shutdown() -> None: - trafficlight.http.adapter.stop_background_tasks = True + adapter.stop_background_tasks = True + await adapter.interrupt_tasks() if kiwi.kiwi_client: await kiwi.kiwi_client.end_run() await adapter_shutdown() + print("Results:\n") + exit_code = 0 + total_tests = 0 + successful_tests = 0 + for testsuite in get_testsuites(): + print( + f"\n{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful" + ) + for testcase in testsuite.test_cases: + print(f" {testcase.client_types}: {testcase.state}") + total_tests += 1 + if testcase.state != "success": + exit_code = 1 + else: + successful_tests = successful_tests + 1 + if testcase.state != "success" and testcase.state != "waiting": + for exception in testcase.exceptions: + print(exception) + + print(f"\nOverall: {successful_tests}/{total_tests} succeeded") + os._exit(exit_code) + return app diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index c9f17a7..6825b8f 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -15,7 +15,7 @@ import asyncio import logging from datetime import datetime, timedelta -from typing import Any, Dict, cast +from typing import Any, Dict, Set, cast from quart import Blueprint, current_app, request from werkzeug.utils import secure_filename @@ -33,6 +33,7 @@ get_adapter, get_adapters, get_tests, + get_testsuites, remove_adapter, ) @@ -63,6 +64,7 @@ async def run() -> None: current_app.add_background_task(run) return + logger.debug( "Not enough client_types to run any test(have %s)", [str(item) for item in available_adapters], @@ -115,21 +117,71 @@ async def cleanup_unresponsive_adapters() -> None: ) +sleeping_tasks: Set[asyncio.Future[None]] = set() + + +async def interrupt_tasks() -> None: + logger.info("Waking up background tasks") + for task in sleeping_tasks: + task.cancel() + + +def should_finish_tests() -> bool: + for testsuite in get_testsuites(): + for testcase in testsuite.test_cases: + if testcase.state not in ("failed", "error", "success"): + logger.info(f"Not exiting because of {testcase}") + return False + return True + + +async def loop_check_all_tests_done() -> None: + while not stop_background_tasks: + logging.debug("Running check for test completion") + if should_finish_tests(): + # do not await because shutdown() awaits all background tasks (inc this one) to shut down first. + asyncio.create_task(current_app.shutdown()) + + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + await sleep_task + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) + logging.info("Termination task shutting down") + + async def loop_cleanup_unresponsive_adapters() -> None: while not stop_background_tasks: - logging.info("Running sweep for idle adapters") + logging.debug("Running sweep for idle adapters") await cleanup_unresponsive_adapters() - await asyncio.sleep(30) - logging.info("Finished sweep task") + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + await sleep_task + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) + logging.info("Sweep task shutting down") async def loop_check_for_new_tests() -> None: while not stop_background_tasks: - logging.info("Running sweep for new tests") + logging.debug("Running sweep for new tests") await check_for_new_tests() - await asyncio.sleep(30) - logging.info("Finished new test task") + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + await sleep_task + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) + logging.info("New test task shutting down") async def adapter_shutdown() -> None: @@ -153,6 +205,7 @@ async def register(adapter_uuid: str): # type: ignore return {} adapter = Adapter(adapter_uuid, registration) add_adapter(adapter) + await interrupt_tasks() return {} diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index 2007223..68003e7 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -242,7 +242,7 @@ async def recreate(self, unload_hooks: bool = False) -> None: async def reload(self) -> None: await self._perform_action({"action": "reload", "data": {}}) - async def create_or_join(self, call_name: str) -> bool: + async def create(self, call_name: str) -> bool: if self.type == self._GUEST_USER: data = await self._perform_action( { @@ -290,6 +290,10 @@ async def get_lobby_data(self) -> LobbyData: snapshot_file = self.test_case.files[self.name + "_" + data["snapshot"]] invite_url = response["data"]["invite_url"] page_url = response["data"]["page_url"] + # Strip trailing & on page URLs until https://github.com/vector-im/element-call/issues/1639 is resolved + if page_url[-1] == "&": + page_url = page_url[1:-1] + call_name = response["data"]["call_name"] lobby_data = LobbyData( video_muted=False, diff --git a/trafficlight/internals/testsuite.py b/trafficlight/internals/testsuite.py index 5d11cde..03fd676 100644 --- a/trafficlight/internals/testsuite.py +++ b/trafficlight/internals/testsuite.py @@ -40,3 +40,16 @@ def waiting(self) -> int: 1 for tc in self.test_cases if tc.state in ("waiting", "preparing") ) return 0 + + def done(self) -> bool: + if self.test_cases is not None: + return ( + sum( + 1 + for tc in self.test_cases + if tc.state in ("waiting", "preparing", "running") + ) + > 0 + ) + else: + return False diff --git a/trafficlight/tests/video/ec_basic_example.py b/trafficlight/tests/video/ec_basic_example.py index 6375a3a..4640b34 100644 --- a/trafficlight/tests/video/ec_basic_example.py +++ b/trafficlight/tests/video/ec_basic_example.py @@ -14,13 +14,10 @@ def __init__(self) -> None: async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - (alice_joined, bob_joined) = await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() - # Check only one of alice or bob joined the room (the other created it) - # between two single-bit booleans, this is xor - print(str(alice_joined) + " or " + str(bob_joined)) + await bob.join_by_url(alice_lobby.invite_url) await asyncio.gather(alice.lobby_join(), bob.lobby_join()) await asyncio.sleep(5) diff --git a/trafficlight/tests/video/handle_invite_base.py b/trafficlight/tests/video/handle_invite_base.py index cb298e4..b91708e 100644 --- a/trafficlight/tests/video/handle_invite_base.py +++ b/trafficlight/tests/video/handle_invite_base.py @@ -15,14 +15,17 @@ async def _run_test( await creator.set_video_image(VideoImage.RED) await joiner.set_video_image(VideoImage.BLUE) - await creator.create_or_join(room_name) + await creator.create(room_name) creator_lobby_data = await creator.get_lobby_data() assert_that(creator_lobby_data.call_name).is_equal_to(room_name) # Now join bob to the call before alice joins the call via page_url - await joiner.join_by_url(creator_lobby_data.page_url) + await joiner.join_by_url(creator_lobby_data.invite_url) + + # For now; wait a little so lobby data settles, because page dynamically updates the page_url + await asyncio.sleep(10) joiner_lobby_data = await joiner.get_lobby_data() diff --git a/trafficlight/tests/video/join_call_recieve_video_test.py b/trafficlight/tests/video/join_call_recieve_video_test.py index 9795c10..21491a7 100644 --- a/trafficlight/tests/video/join_call_recieve_video_test.py +++ b/trafficlight/tests/video/join_call_recieve_video_test.py @@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) # lobby screen await asyncio.gather(alice.lobby_join(), bob.lobby_join()) await asyncio.sleep(5) diff --git a/trafficlight/tests/video/load_test_call_test.py b/trafficlight/tests/video/load_test_call_test.py index 898a575..56c9eee 100644 --- a/trafficlight/tests/video/load_test_call_test.py +++ b/trafficlight/tests/video/load_test_call_test.py @@ -25,7 +25,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) # Create room - await alice.create_or_join(room_name) + await alice.create(room_name) lobby_data = await alice.get_lobby_data() diff --git a/trafficlight/tests/video/three_user_spotlight.py b/trafficlight/tests/video/three_user_spotlight.py index 3705f47..428b869 100644 --- a/trafficlight/tests/video/three_user_spotlight.py +++ b/trafficlight/tests/video/three_user_spotlight.py @@ -19,9 +19,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) # lobby screen await asyncio.gather(alice.lobby_join(), bob.lobby_join()) diff --git a/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py b/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py index 67482e6..f8af4a6 100644 --- a/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py +++ b/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py @@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) await alice.set_video_image(VideoImage.BLUE) await bob.set_video_image(VideoImage.RED) @@ -53,7 +54,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: await bob.set_video_image(VideoImage.GREEN) - await bob.create_or_join(room_name) + await bob.join_by_url(alice_data.invite_url) await bob.lobby_join()