From f0a127993805a1bd38c32c4ca86353898194b215 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:50:16 +0100 Subject: [PATCH 1/9] Refactor to allow cancellation of sleeps, and react to shutdown. --- trafficlight/__init__.py | 1 + trafficlight/http/adapter.py | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 5092900..c2ac367 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -120,6 +120,7 @@ async def startup() -> None: @app.after_serving async def shutdown() -> None: trafficlight.http.adapter.stop_background_tasks = True + await trafficlight.http.adapter.interrupt_tasks() if kiwi.kiwi_client: await kiwi.kiwi_client.end_run() await adapter_shutdown() diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index c9f17a7..f8183fa 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -15,7 +15,7 @@ import asyncio import logging from datetime import datetime, timedelta -from typing import Any, Dict, cast +from typing import Any, Dict, Set, cast from quart import Blueprint, current_app, request from werkzeug.utils import secure_filename @@ -63,6 +63,7 @@ async def run() -> None: current_app.add_background_task(run) return + logger.debug( "Not enough client_types to run any test(have %s)", [str(item) for item in available_adapters], @@ -115,12 +116,26 @@ async def cleanup_unresponsive_adapters() -> None: ) +sleeping_tasks: Set[asyncio.Future[None]] = set() + + +async def interrupt_tasks() -> None: + for task in sleeping_tasks: + task.cancel() + + async def loop_cleanup_unresponsive_adapters() -> None: while not stop_background_tasks: logging.info("Running sweep for idle adapters") await cleanup_unresponsive_adapters() - await asyncio.sleep(30) + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) logging.info("Finished sweep task") @@ -128,7 +143,13 @@ async def loop_check_for_new_tests() -> None: while not stop_background_tasks: logging.info("Running sweep for new tests") await check_for_new_tests() - await asyncio.sleep(30) + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) logging.info("Finished new test task") From 59306a7fc2ed884243b3f4a6fdcef4056c99d55f Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:09:01 +0100 Subject: [PATCH 2/9] On shutdown (of any kind) log summary of test cases. On all tests completing or failing, shutdown automatically. --- trafficlight/__init__.py | 26 ++++++++++++++++++++++---- trafficlight/http/adapter.py | 2 ++ trafficlight/internals/testcase.py | 5 ++++- trafficlight/internals/testsuite.py | 3 +++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index c2ac367..550070a 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -12,25 +12,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import json import logging import os import uuid +from asyncio import Future from datetime import datetime, timedelta from typing import Any, Dict, Optional from quart import Quart -import trafficlight import trafficlight.kiwi as kiwi from trafficlight.homerunner import HomerunnerClient from trafficlight.http.adapter import ( adapter_shutdown, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, + stop_background_tasks, + interrupt_tasks, ) from trafficlight.internals.testsuite import TestSuite -from trafficlight.store import add_testsuite +from trafficlight.store import add_testsuite, get_testsuites from trafficlight.tests import load_tests logger = logging.getLogger(__name__) @@ -110,19 +113,34 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: ) app.jinja_env.filters["delaytime"] = format_delaytime + async def on_done(f: Future) -> None: + + await app.shutdown() + @app.before_serving async def startup() -> None: app.add_background_task(loop_cleanup_unresponsive_adapters) app.add_background_task(loop_check_for_new_tests) if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() + finished = asyncio.gather(*list(map(lambda x: x.completed, get_testsuites()))) + finished.add_done_callback(on_done) @app.after_serving async def shutdown() -> None: - trafficlight.http.adapter.stop_background_tasks = True - await trafficlight.http.adapter.interrupt_tasks() + adapter.stop_background_tasks = True + await adapter.interrupt_tasks() if kiwi.kiwi_client: await kiwi.kiwi_client.end_run() await adapter_shutdown() + print("Results:\n\n") + for testsuite in get_testsuites(): + print( + f"{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful" + ) + for testcase in testsuite.test_cases: + print(f" {testcase.client_types}: {testcase.state}") + if testcase.state != "success" and testcase.state != "waiting": + print(f"{testcase.exceptions}") return app diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index f8183fa..8626b90 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -132,6 +132,7 @@ async def loop_cleanup_unresponsive_adapters() -> None: sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) try: sleeping_tasks.add(sleep_task) + await sleep_task except asyncio.CancelledError: pass # we don't mind this task being cancelled. finally: @@ -146,6 +147,7 @@ async def loop_check_for_new_tests() -> None: sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) try: sleeping_tasks.add(sleep_task) + await sleep_task except asyncio.CancelledError: pass # we don't mind this task being cancelled. finally: diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index 578073d..0b2311e 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -1,3 +1,4 @@ +import asyncio import hashlib import logging import traceback @@ -40,6 +41,7 @@ def __init__( self.servers: List[HomeServer] = [] self.files: Dict[str, str] = {} self.adapters: Optional[Dict[str, Adapter]] = None + self.completed = asyncio.get_event_loop().create_future() def __repr__(self) -> str: return f"{self.test.name()} ({self.server_type} {self.client_types})" @@ -111,7 +113,7 @@ async def run( # Treating an adapter that causes another type of exception as an error self.state = "error" self.exceptions.append(e.formatted_message) - except Exception: + except Exception as e: # Treating everything else as an error as well... eg compilation failures self.state = "error" self.exceptions.append("".join(traceback.format_exc())) @@ -122,3 +124,4 @@ async def run( server.finished() if kiwi.kiwi_client: await kiwi.kiwi_client.report_status(self) + self.completed.set_result(True) diff --git a/trafficlight/internals/testsuite.py b/trafficlight/internals/testsuite.py index 5d11cde..7589710 100644 --- a/trafficlight/internals/testsuite.py +++ b/trafficlight/internals/testsuite.py @@ -1,3 +1,4 @@ +import asyncio import hashlib from typing import List @@ -10,6 +11,8 @@ def __init__(self, test: Test, test_cases: List[TestCase]): self.guid = hashlib.md5(f"TestSuite{test.name()}".encode("utf-8")).hexdigest() self.test = test self.test_cases = test_cases + futures = list(map(lambda c: c.completed, test_cases)) + self.completed = asyncio.gather(*futures) def name(self) -> str: return self.test.name() From 479ad5e67cefa1df0fd90327b2c828bf707559cb Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:35:33 +0100 Subject: [PATCH 3/9] linting --- trafficlight/__init__.py | 5 +---- trafficlight/internals/testcase.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 550070a..3f0d40b 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -29,8 +29,6 @@ adapter_shutdown, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, - stop_background_tasks, - interrupt_tasks, ) from trafficlight.internals.testsuite import TestSuite from trafficlight.store import add_testsuite, get_testsuites @@ -113,8 +111,7 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: ) app.jinja_env.filters["delaytime"] = format_delaytime - async def on_done(f: Future) -> None: - + async def on_done(f: Future[Any]) -> None: await app.shutdown() @app.before_serving diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index 0b2311e..a5f524a 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -113,7 +113,7 @@ async def run( # Treating an adapter that causes another type of exception as an error self.state = "error" self.exceptions.append(e.formatted_message) - except Exception as e: + except Exception: # Treating everything else as an error as well... eg compilation failures self.state = "error" self.exceptions.append("".join(traceback.format_exc())) From 3f82d6135318271fae87ce79b8df0f7f8744fd25 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:41:55 +0100 Subject: [PATCH 4/9] Continue work --- trafficlight/__init__.py | 19 +++++++++++++------ trafficlight/http/adapter.py | 2 ++ trafficlight/internals/client.py | 2 +- trafficlight/internals/testcase.py | 7 ++++++- trafficlight/internals/testsuite.py | 13 +++++++++++-- 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 3f0d40b..89bf31f 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -21,7 +21,7 @@ from datetime import datetime, timedelta from typing import Any, Dict, Optional -from quart import Quart +from quart import Quart, current_app import trafficlight.kiwi as kiwi from trafficlight.homerunner import HomerunnerClient @@ -111,17 +111,23 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: ) app.jinja_env.filters["delaytime"] = format_delaytime - async def on_done(f: Future[Any]) -> None: - await app.shutdown() - @app.before_serving async def startup() -> None: app.add_background_task(loop_cleanup_unresponsive_adapters) app.add_background_task(loop_check_for_new_tests) if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() - finished = asyncio.gather(*list(map(lambda x: x.completed, get_testsuites()))) - finished.add_done_callback(on_done) + + async def wait_for_done() -> None: + try: + if suite.done() for suite in get_testsuites(): + if suite.done() + logger.info("Everything done!") + await app.shutdown() + + app.add_background_task( + wait_for_done, + ) @app.after_serving async def shutdown() -> None: @@ -130,6 +136,7 @@ async def shutdown() -> None: if kiwi.kiwi_client: await kiwi.kiwi_client.end_run() await adapter_shutdown() + print("Results:\n\n") for testsuite in get_testsuites(): print( diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index 8626b90..d0d69a3 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -120,6 +120,7 @@ async def cleanup_unresponsive_adapters() -> None: async def interrupt_tasks() -> None: + logger.info("Waking up background tasks") for task in sleeping_tasks: task.cancel() @@ -176,6 +177,7 @@ async def register(adapter_uuid: str): # type: ignore return {} adapter = Adapter(adapter_uuid, registration) add_adapter(adapter) + await interrupt_tasks() return {} diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index 2007223..2bf42ce 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -242,7 +242,7 @@ async def recreate(self, unload_hooks: bool = False) -> None: async def reload(self) -> None: await self._perform_action({"action": "reload", "data": {}}) - async def create_or_join(self, call_name: str) -> bool: + async def create(self, call_name: str) -> bool: if self.type == self._GUEST_USER: data = await self._perform_action( { diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index a5f524a..0ac81a8 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -2,6 +2,7 @@ import hashlib import logging import traceback +from asyncio import Future from typing import Any, Dict, List, Optional, Union import trafficlight.kiwi as kiwi @@ -27,6 +28,7 @@ def __init__( server_names: List[str], client_types: Dict[str, ClientType], ) -> None: + self.completed: Future[None] self.exceptions: List[str] = [] self.guid = hashlib.md5( f"TestCase{test.name()}{server_type.name() if server_type else None}{server_names}{client_types}".encode( @@ -41,7 +43,6 @@ def __init__( self.servers: List[HomeServer] = [] self.files: Dict[str, str] = {} self.adapters: Optional[Dict[str, Adapter]] = None - self.completed = asyncio.get_event_loop().create_future() def __repr__(self) -> str: return f"{self.test.name()} ({self.server_type} {self.client_types})" @@ -125,3 +126,7 @@ async def run( if kiwi.kiwi_client: await kiwi.kiwi_client.report_status(self) self.completed.set_result(True) + + async def wait_for_completion(self) -> None: + self.completed = asyncio.get_running_loop().create_future() + await self.completed diff --git a/trafficlight/internals/testsuite.py b/trafficlight/internals/testsuite.py index 7589710..35428ea 100644 --- a/trafficlight/internals/testsuite.py +++ b/trafficlight/internals/testsuite.py @@ -11,8 +11,6 @@ def __init__(self, test: Test, test_cases: List[TestCase]): self.guid = hashlib.md5(f"TestSuite{test.name()}".encode("utf-8")).hexdigest() self.test = test self.test_cases = test_cases - futures = list(map(lambda c: c.completed, test_cases)) - self.completed = asyncio.gather(*futures) def name(self) -> str: return self.test.name() @@ -43,3 +41,14 @@ def waiting(self) -> int: 1 for tc in self.test_cases if tc.state in ("waiting", "preparing") ) return 0 + + def done(self) -> bool: + if self.test_cases is not None: + return ( + sum( + 1 + for tc in self.test_cases + if tc.state in ("waiting", "preparing", "running") + ) + > 0 + ) From 14efa47e5017f81a0364f4cf40cb3f3b5e7c352b Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:42:02 +0100 Subject: [PATCH 5/9] Update EC tests - we can nolonger join by room name. --- trafficlight/tests/video/handle_invite_base.py | 4 ++-- trafficlight/tests/video/join_call_recieve_video_test.py | 7 ++++--- .../video/two_clients_one_terminated_rejoin_test.py | 9 +++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/trafficlight/tests/video/handle_invite_base.py b/trafficlight/tests/video/handle_invite_base.py index cb298e4..cf3ebc4 100644 --- a/trafficlight/tests/video/handle_invite_base.py +++ b/trafficlight/tests/video/handle_invite_base.py @@ -15,14 +15,14 @@ async def _run_test( await creator.set_video_image(VideoImage.RED) await joiner.set_video_image(VideoImage.BLUE) - await creator.create_or_join(room_name) + await creator.create(room_name) creator_lobby_data = await creator.get_lobby_data() assert_that(creator_lobby_data.call_name).is_equal_to(room_name) # Now join bob to the call before alice joins the call via page_url - await joiner.join_by_url(creator_lobby_data.page_url) + await joiner.join_by_url(creator_lobby_data.invite_url) joiner_lobby_data = await joiner.get_lobby_data() diff --git a/trafficlight/tests/video/join_call_recieve_video_test.py b/trafficlight/tests/video/join_call_recieve_video_test.py index 9795c10..21491a7 100644 --- a/trafficlight/tests/video/join_call_recieve_video_test.py +++ b/trafficlight/tests/video/join_call_recieve_video_test.py @@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) # lobby screen await asyncio.gather(alice.lobby_join(), bob.lobby_join()) await asyncio.sleep(5) diff --git a/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py b/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py index 67482e6..f8af4a6 100644 --- a/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py +++ b/trafficlight/tests/video/two_clients_one_terminated_rejoin_test.py @@ -23,9 +23,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) await alice.set_video_image(VideoImage.BLUE) await bob.set_video_image(VideoImage.RED) @@ -53,7 +54,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: await bob.set_video_image(VideoImage.GREEN) - await bob.create_or_join(room_name) + await bob.join_by_url(alice_data.invite_url) await bob.lobby_join() From 95a398f7b6a22cf0e2fecba369cd9e642f1ee74f Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:11:25 +0100 Subject: [PATCH 6/9] Remove accidental commmit --- trafficlight/__init__.py | 15 +-------------- trafficlight/internals/testcase.py | 4 ++-- trafficlight/internals/testsuite.py | 3 ++- trafficlight/tests/video/ec_basic_example.py | 9 +++------ trafficlight/tests/video/load_test_call_test.py | 2 +- trafficlight/tests/video/three_user_spotlight.py | 7 ++++--- 6 files changed, 13 insertions(+), 27 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 89bf31f..2f7cad9 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -12,16 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import json import logging import os import uuid -from asyncio import Future from datetime import datetime, timedelta from typing import Any, Dict, Optional -from quart import Quart, current_app +from quart import Quart import trafficlight.kiwi as kiwi from trafficlight.homerunner import HomerunnerClient @@ -118,17 +116,6 @@ async def startup() -> None: if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() - async def wait_for_done() -> None: - try: - if suite.done() for suite in get_testsuites(): - if suite.done() - logger.info("Everything done!") - await app.shutdown() - - app.add_background_task( - wait_for_done, - ) - @app.after_serving async def shutdown() -> None: adapter.stop_background_tasks = True diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index 0ac81a8..147bb95 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -28,7 +28,7 @@ def __init__( server_names: List[str], client_types: Dict[str, ClientType], ) -> None: - self.completed: Future[None] + self.completed: Optional[Future[bool]] = None self.exceptions: List[str] = [] self.guid = hashlib.md5( f"TestCase{test.name()}{server_type.name() if server_type else None}{server_names}{client_types}".encode( @@ -125,7 +125,7 @@ async def run( server.finished() if kiwi.kiwi_client: await kiwi.kiwi_client.report_status(self) - self.completed.set_result(True) + self.completed.set_result(False) async def wait_for_completion(self) -> None: self.completed = asyncio.get_running_loop().create_future() diff --git a/trafficlight/internals/testsuite.py b/trafficlight/internals/testsuite.py index 35428ea..03fd676 100644 --- a/trafficlight/internals/testsuite.py +++ b/trafficlight/internals/testsuite.py @@ -1,4 +1,3 @@ -import asyncio import hashlib from typing import List @@ -52,3 +51,5 @@ def done(self) -> bool: ) > 0 ) + else: + return False diff --git a/trafficlight/tests/video/ec_basic_example.py b/trafficlight/tests/video/ec_basic_example.py index 6375a3a..4640b34 100644 --- a/trafficlight/tests/video/ec_basic_example.py +++ b/trafficlight/tests/video/ec_basic_example.py @@ -14,13 +14,10 @@ def __init__(self) -> None: async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - (alice_joined, bob_joined) = await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() - # Check only one of alice or bob joined the room (the other created it) - # between two single-bit booleans, this is xor - print(str(alice_joined) + " or " + str(bob_joined)) + await bob.join_by_url(alice_lobby.invite_url) await asyncio.gather(alice.lobby_join(), bob.lobby_join()) await asyncio.sleep(5) diff --git a/trafficlight/tests/video/load_test_call_test.py b/trafficlight/tests/video/load_test_call_test.py index 898a575..56c9eee 100644 --- a/trafficlight/tests/video/load_test_call_test.py +++ b/trafficlight/tests/video/load_test_call_test.py @@ -25,7 +25,7 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) # Create room - await alice.create_or_join(room_name) + await alice.create(room_name) lobby_data = await alice.get_lobby_data() diff --git a/trafficlight/tests/video/three_user_spotlight.py b/trafficlight/tests/video/three_user_spotlight.py index 3705f47..428b869 100644 --- a/trafficlight/tests/video/three_user_spotlight.py +++ b/trafficlight/tests/video/three_user_spotlight.py @@ -19,9 +19,10 @@ async def run(self, alice: ElementCallClient, bob: ElementCallClient) -> None: room_name = "tl_chat_" + str(datetime.now().timestamp()) - await asyncio.gather( - alice.create_or_join(room_name), bob.create_or_join(room_name) - ) + await alice.create(room_name) + alice_lobby = await alice.get_lobby_data() + + await bob.join_by_url(alice_lobby.invite_url) # lobby screen await asyncio.gather(alice.lobby_join(), bob.lobby_join()) From b1532e6947973f8ab4e51215f80d54041ba05d24 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:52:14 +0100 Subject: [PATCH 7/9] Roll back incorrect approach --- trafficlight/internals/testcase.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index 147bb95..578073d 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -1,8 +1,6 @@ -import asyncio import hashlib import logging import traceback -from asyncio import Future from typing import Any, Dict, List, Optional, Union import trafficlight.kiwi as kiwi @@ -28,7 +26,6 @@ def __init__( server_names: List[str], client_types: Dict[str, ClientType], ) -> None: - self.completed: Optional[Future[bool]] = None self.exceptions: List[str] = [] self.guid = hashlib.md5( f"TestCase{test.name()}{server_type.name() if server_type else None}{server_names}{client_types}".encode( @@ -125,8 +122,3 @@ async def run( server.finished() if kiwi.kiwi_client: await kiwi.kiwi_client.report_status(self) - self.completed.set_result(False) - - async def wait_for_completion(self) -> None: - self.completed = asyncio.get_running_loop().create_future() - await self.completed From e3add30dd9388edfeeb55b4e85d2238a32a6e3fe Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Thu, 28 Sep 2023 15:21:12 +0100 Subject: [PATCH 8/9] Finally sort out the flow --- trafficlight/__init__.py | 20 +++++++++-- trafficlight/http/adapter.py | 36 ++++++++++++++++--- trafficlight/internals/client.py | 4 +++ .../tests/video/handle_invite_base.py | 3 ++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 2f7cad9..04af2f5 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -27,6 +27,7 @@ adapter_shutdown, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, + loop_check_all_tests_done, ) from trafficlight.internals.testsuite import TestSuite from trafficlight.store import add_testsuite, get_testsuites @@ -113,6 +114,7 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: async def startup() -> None: app.add_background_task(loop_cleanup_unresponsive_adapters) app.add_background_task(loop_check_for_new_tests) + app.add_background_task(loop_check_all_tests_done) if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() @@ -124,14 +126,26 @@ async def shutdown() -> None: await kiwi.kiwi_client.end_run() await adapter_shutdown() - print("Results:\n\n") + print("Results:\n") + exit_code = 0 + total_tests = 0 + successful_tests = 0 for testsuite in get_testsuites(): print( - f"{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful" + f"\n{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful" ) for testcase in testsuite.test_cases: print(f" {testcase.client_types}: {testcase.state}") + total_tests += 1 + if testcase.state != "success": + exit_code = 1 + else: + successful_tests = successful_tests + 1 if testcase.state != "success" and testcase.state != "waiting": - print(f"{testcase.exceptions}") + for exception in testcase.exceptions: + print(exception) + + print(f"\nOverall: {successful_tests}/{total_tests} succeeded") + os._exit(exit_code) return app diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index d0d69a3..fb7057c 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -34,6 +34,7 @@ get_adapters, get_tests, remove_adapter, + get_testsuites, ) IDLE_ADAPTER_UNRESPONSIVE_DELAY = timedelta(minutes=1) @@ -125,9 +126,36 @@ async def interrupt_tasks() -> None: task.cancel() +def should_finish_tests() -> bool: + for testsuite in get_testsuites(): + for testcase in testsuite.test_cases: + if testcase.state not in ("failed", "error", "success"): + logger.info(f"Not exiting because of {testcase}") + return False + return True + + +async def loop_check_all_tests_done() -> None: + while not stop_background_tasks: + logging.debug("Running check for test completion") + if should_finish_tests(): + # do not await because shutdown() awaits all background tasks (inc this one) to shut down first. + asyncio.create_task(current_app.shutdown()) + + sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) + try: + sleeping_tasks.add(sleep_task) + await sleep_task + except asyncio.CancelledError: + pass # we don't mind this task being cancelled. + finally: + sleeping_tasks.remove(sleep_task) + logging.info("Termination task shutting down") + + async def loop_cleanup_unresponsive_adapters() -> None: while not stop_background_tasks: - logging.info("Running sweep for idle adapters") + logging.debug("Running sweep for idle adapters") await cleanup_unresponsive_adapters() sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) @@ -138,12 +166,12 @@ async def loop_cleanup_unresponsive_adapters() -> None: pass # we don't mind this task being cancelled. finally: sleeping_tasks.remove(sleep_task) - logging.info("Finished sweep task") + logging.info("Sweep task shutting down") async def loop_check_for_new_tests() -> None: while not stop_background_tasks: - logging.info("Running sweep for new tests") + logging.debug("Running sweep for new tests") await check_for_new_tests() sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) try: @@ -153,7 +181,7 @@ async def loop_check_for_new_tests() -> None: pass # we don't mind this task being cancelled. finally: sleeping_tasks.remove(sleep_task) - logging.info("Finished new test task") + logging.info("New test task shutting down") async def adapter_shutdown() -> None: diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index 2bf42ce..68003e7 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -290,6 +290,10 @@ async def get_lobby_data(self) -> LobbyData: snapshot_file = self.test_case.files[self.name + "_" + data["snapshot"]] invite_url = response["data"]["invite_url"] page_url = response["data"]["page_url"] + # Strip trailing & on page URLs until https://github.com/vector-im/element-call/issues/1639 is resolved + if page_url[-1] == "&": + page_url = page_url[1:-1] + call_name = response["data"]["call_name"] lobby_data = LobbyData( video_muted=False, diff --git a/trafficlight/tests/video/handle_invite_base.py b/trafficlight/tests/video/handle_invite_base.py index cf3ebc4..b91708e 100644 --- a/trafficlight/tests/video/handle_invite_base.py +++ b/trafficlight/tests/video/handle_invite_base.py @@ -24,6 +24,9 @@ async def _run_test( await joiner.join_by_url(creator_lobby_data.invite_url) + # For now; wait a little so lobby data settles, because page dynamically updates the page_url + await asyncio.sleep(10) + joiner_lobby_data = await joiner.get_lobby_data() with soft_assertions(): From 80d5437be406167b9c07e054f708df624924bb9f Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:15:59 +0100 Subject: [PATCH 9/9] isort --- trafficlight/__init__.py | 2 +- trafficlight/http/adapter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index 04af2f5..da23d01 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -25,9 +25,9 @@ from trafficlight.homerunner import HomerunnerClient from trafficlight.http.adapter import ( adapter_shutdown, + loop_check_all_tests_done, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, - loop_check_all_tests_done, ) from trafficlight.internals.testsuite import TestSuite from trafficlight.store import add_testsuite, get_testsuites diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index fb7057c..6825b8f 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -33,8 +33,8 @@ get_adapter, get_adapters, get_tests, - remove_adapter, get_testsuites, + remove_adapter, ) IDLE_ADAPTER_UNRESPONSIVE_DELAY = timedelta(minutes=1)