Merge pull request #86 from matrix-org/michaelk/refactor_style_of_run…

…ning

Refactor style of running

trafficlight/__init__.py

@@ -21,16 +21,16 @@
 
 from quart import Quart
 
-import trafficlight
 import trafficlight.kiwi as kiwi
 from trafficlight.homerunner import HomerunnerClient
 from trafficlight.http.adapter import (
     adapter_shutdown,
+    loop_check_all_tests_done,
     loop_check_for_new_tests,
     loop_cleanup_unresponsive_adapters,
 )
 from trafficlight.internals.testsuite import TestSuite
-from trafficlight.store import add_testsuite
+from trafficlight.store import add_testsuite, get_testsuites
 from trafficlight.tests import load_tests
 
 logger = logging.getLogger(__name__)
@@ -114,14 +114,38 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart:
     async def startup() -> None:
         app.add_background_task(loop_cleanup_unresponsive_adapters)
         app.add_background_task(loop_check_for_new_tests)
+        app.add_background_task(loop_check_all_tests_done)
         if kiwi.kiwi_client:
             await kiwi.kiwi_client.start_run()
 
     @app.after_serving
     async def shutdown() -> None:
-        trafficlight.http.adapter.stop_background_tasks = True
+        adapter.stop_background_tasks = True
+        await adapter.interrupt_tasks()
         if kiwi.kiwi_client:
             await kiwi.kiwi_client.end_run()
         await adapter_shutdown()
 
+        print("Results:\n")
+        exit_code = 0
+        total_tests = 0
+        successful_tests = 0
+        for testsuite in get_testsuites():
+            print(
+                f"\n{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful"
+            )
+            for testcase in testsuite.test_cases:
+                print(f"  {testcase.client_types}: {testcase.state}")
+                total_tests += 1
+                if testcase.state != "success":
+                    exit_code = 1
+                else:
+                    successful_tests = successful_tests + 1
+                if testcase.state != "success" and testcase.state != "waiting":
+                    for exception in testcase.exceptions:
+                        print(exception)
+
+        print(f"\nOverall: {successful_tests}/{total_tests} succeeded")
+        os._exit(exit_code)
+
     return app

trafficlight/http/adapter.py

@@ -15,7 +15,7 @@
 import asyncio
 import logging
 from datetime import datetime, timedelta
-from typing import Any, Dict, cast
+from typing import Any, Dict, Set, cast
 
 from quart import Blueprint, current_app, request
 from werkzeug.utils import secure_filename
@@ -33,6 +33,7 @@
     get_adapter,
     get_adapters,
     get_tests,
+    get_testsuites,
     remove_adapter,
 )
 
@@ -63,6 +64,7 @@ async def run() -> None:
 
                 current_app.add_background_task(run)
                 return
+
     logger.debug(
         "Not enough client_types to run any test(have %s)",
         [str(item) for item in available_adapters],
@@ -115,21 +117,71 @@ async def cleanup_unresponsive_adapters() -> None:
                 )
 
 
+sleeping_tasks: Set[asyncio.Future[None]] = set()
+
+
+async def interrupt_tasks() -> None:
+    logger.info("Waking up background tasks")
+    for task in sleeping_tasks:
+        task.cancel()
+
+
+def should_finish_tests() -> bool:
+    for testsuite in get_testsuites():
+        for testcase in testsuite.test_cases:
+            if testcase.state not in ("failed", "error", "success"):
+                logger.info(f"Not exiting because of {testcase}")
+                return False
+    return True
+
+
+async def loop_check_all_tests_done() -> None:
+    while not stop_background_tasks:
+        logging.debug("Running check for test completion")
+        if should_finish_tests():
+            # do not await because shutdown() awaits all background tasks (inc this one) to shut down first.
+            asyncio.create_task(current_app.shutdown())
+
+        sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
+        try:
+            sleeping_tasks.add(sleep_task)
+            await sleep_task
+        except asyncio.CancelledError:
+            pass  # we don't mind this task being cancelled.
+        finally:
+            sleeping_tasks.remove(sleep_task)
+    logging.info("Termination task shutting down")
+
+
 async def loop_cleanup_unresponsive_adapters() -> None:
     while not stop_background_tasks:
-        logging.info("Running sweep for idle adapters")
+        logging.debug("Running sweep for idle adapters")
         await cleanup_unresponsive_adapters()
-        await asyncio.sleep(30)
 
-    logging.info("Finished sweep task")
+        sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
+        try:
+            sleeping_tasks.add(sleep_task)
+            await sleep_task
+        except asyncio.CancelledError:
+            pass  # we don't mind this task being cancelled.
+        finally:
+            sleeping_tasks.remove(sleep_task)
+    logging.info("Sweep task shutting down")
 
 
 async def loop_check_for_new_tests() -> None:
     while not stop_background_tasks:
-        logging.info("Running sweep for new tests")
+        logging.debug("Running sweep for new tests")
         await check_for_new_tests()
-        await asyncio.sleep(30)
-    logging.info("Finished new test task")
+        sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30))
+        try:
+            sleeping_tasks.add(sleep_task)
+            await sleep_task
+        except asyncio.CancelledError:
+            pass  # we don't mind this task being cancelled.
+        finally:
+            sleeping_tasks.remove(sleep_task)
+    logging.info("New test task shutting down")
 
 
 async def adapter_shutdown() -> None:
@@ -153,6 +205,7 @@ async def register(adapter_uuid: str):  # type: ignore
             return {}
     adapter = Adapter(adapter_uuid, registration)
     add_adapter(adapter)
+    await interrupt_tasks()
     return {}
 
 

trafficlight/internals/client.py

@@ -242,7 +242,7 @@ async def recreate(self, unload_hooks: bool = False) -> None:
     async def reload(self) -> None:
         await self._perform_action({"action": "reload", "data": {}})
 
-    async def create_or_join(self, call_name: str) -> bool:
+    async def create(self, call_name: str) -> bool:
         if self.type == self._GUEST_USER:
             data = await self._perform_action(
                 {
@@ -290,6 +290,10 @@ async def get_lobby_data(self) -> LobbyData:
         snapshot_file = self.test_case.files[self.name + "_" + data["snapshot"]]
         invite_url = response["data"]["invite_url"]
         page_url = response["data"]["page_url"]
+        # Strip trailing & on page URLs until https://github.com/vector-im/element-call/issues/1639 is resolved
+        if page_url[-1] == "&":
+            page_url = page_url[1:-1]
+
         call_name = response["data"]["call_name"]
         lobby_data = LobbyData(
             video_muted=False,

trafficlight/internals/testsuite.py

@@ -40,3 +40,16 @@ def waiting(self) -> int:
                 1 for tc in self.test_cases if tc.state in ("waiting", "preparing")
             )
         return 0
+
+    def done(self) -> bool:
+        if self.test_cases is not None:
+            return (
+                sum(
+                    1
+                    for tc in self.test_cases
+                    if tc.state in ("waiting", "preparing", "running")
+                )
+                > 0
+            )
+        else:
+            return False

trafficlight/tests/video/ec_basic_example.py

@@ -14,13 +14,10 @@ def __init__(self) -> None:
 
     async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
         room_name = "tl_chat_" + str(datetime.now().timestamp())
-        (alice_joined, bob_joined) = await asyncio.gather(
-            alice.create_or_join(room_name), bob.create_or_join(room_name)
-        )
+        await alice.create(room_name)
+        alice_lobby = await alice.get_lobby_data()
 
-        # Check only one of alice or bob joined the room (the other created it)
-        # between two single-bit booleans, this is xor
-        print(str(alice_joined) + " or " + str(bob_joined))
+        await bob.join_by_url(alice_lobby.invite_url)
 
         await asyncio.gather(alice.lobby_join(), bob.lobby_join())
         await asyncio.sleep(5)

trafficlight/tests/video/handle_invite_base.py

@@ -15,14 +15,17 @@ async def _run_test(
         await creator.set_video_image(VideoImage.RED)
         await joiner.set_video_image(VideoImage.BLUE)
 
-        await creator.create_or_join(room_name)
+        await creator.create(room_name)
 
         creator_lobby_data = await creator.get_lobby_data()
         assert_that(creator_lobby_data.call_name).is_equal_to(room_name)
 
         # Now join bob to the call before alice joins the call via page_url
 
-        await joiner.join_by_url(creator_lobby_data.page_url)
+        await joiner.join_by_url(creator_lobby_data.invite_url)
+
+        # For now; wait a little so lobby data settles, because page dynamically updates the page_url
+        await asyncio.sleep(10)
 
         joiner_lobby_data = await joiner.get_lobby_data()
 

trafficlight/tests/video/join_call_recieve_video_test.py

@@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
 
         room_name = "tl_chat_" + str(datetime.now().timestamp())
 
-        await asyncio.gather(
-            alice.create_or_join(room_name), bob.create_or_join(room_name)
-        )
+        await alice.create(room_name)
+        alice_lobby = await alice.get_lobby_data()
+
+        await bob.join_by_url(alice_lobby.invite_url)
         # lobby screen
         await asyncio.gather(alice.lobby_join(), bob.lobby_join())
         await asyncio.sleep(5)

trafficlight/tests/video/load_test_call_test.py

@@ -25,7 +25,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
         room_name = "tl_chat_" + str(datetime.now().timestamp())
 
         # Create room
-        await alice.create_or_join(room_name)
+        await alice.create(room_name)
 
         lobby_data = await alice.get_lobby_data()
 

trafficlight/tests/video/three_user_spotlight.py

@@ -19,9 +19,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
 
         room_name = "tl_chat_" + str(datetime.now().timestamp())
 
-        await asyncio.gather(
-            alice.create_or_join(room_name), bob.create_or_join(room_name)
-        )
+        await alice.create(room_name)
+        alice_lobby = await alice.get_lobby_data()
+
+        await bob.join_by_url(alice_lobby.invite_url)
 
         # lobby screen
         await asyncio.gather(alice.lobby_join(), bob.lobby_join())

trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py

@@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
 
         room_name = "tl_chat_" + str(datetime.now().timestamp())
 
-        await asyncio.gather(
-            alice.create_or_join(room_name), bob.create_or_join(room_name)
-        )
+        await alice.create(room_name)
+        alice_lobby = await alice.get_lobby_data()
+
+        await bob.join_by_url(alice_lobby.invite_url)
 
         await alice.set_video_image(VideoImage.BLUE)
         await bob.set_video_image(VideoImage.RED)
@@ -53,7 +54,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None:
 
         await bob.set_video_image(VideoImage.GREEN)
 
-        await bob.create_or_join(room_name)
+        await bob.join_by_url(alice_data.invite_url)
 
         await bob.lobby_join()