From 59306a7fc2ed884243b3f4a6fdcef4056c99d55f Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:09:01 +0100 Subject: [PATCH] On shutdown (of any kind) log summary of test cases. On all tests completing or failing, shutdown automatically. --- trafficlight/__init__.py | 26 ++++++++++++++++++++++---- trafficlight/http/adapter.py | 2 ++ trafficlight/internals/testcase.py | 5 ++++- trafficlight/internals/testsuite.py | 3 +++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/trafficlight/__init__.py b/trafficlight/__init__.py index c2ac367..550070a 100644 --- a/trafficlight/__init__.py +++ b/trafficlight/__init__.py @@ -12,25 +12,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import json import logging import os import uuid +from asyncio import Future from datetime import datetime, timedelta from typing import Any, Dict, Optional from quart import Quart -import trafficlight import trafficlight.kiwi as kiwi from trafficlight.homerunner import HomerunnerClient from trafficlight.http.adapter import ( adapter_shutdown, loop_check_for_new_tests, loop_cleanup_unresponsive_adapters, + stop_background_tasks, + interrupt_tasks, ) from trafficlight.internals.testsuite import TestSuite -from trafficlight.store import add_testsuite +from trafficlight.store import add_testsuite, get_testsuites from trafficlight.tests import load_tests logger = logging.getLogger(__name__) @@ -110,19 +113,34 @@ def create_app(test_config: Optional[Dict[str, Any]] = None) -> Quart: ) app.jinja_env.filters["delaytime"] = format_delaytime + async def on_done(f: Future) -> None: + + await app.shutdown() + @app.before_serving async def startup() -> None: app.add_background_task(loop_cleanup_unresponsive_adapters) app.add_background_task(loop_check_for_new_tests) if kiwi.kiwi_client: await kiwi.kiwi_client.start_run() + finished = asyncio.gather(*list(map(lambda x: x.completed, get_testsuites()))) + finished.add_done_callback(on_done) @app.after_serving async def shutdown() -> None: - trafficlight.http.adapter.stop_background_tasks = True - await trafficlight.http.adapter.interrupt_tasks() + adapter.stop_background_tasks = True + await adapter.interrupt_tasks() if kiwi.kiwi_client: await kiwi.kiwi_client.end_run() await adapter_shutdown() + print("Results:\n\n") + for testsuite in get_testsuites(): + print( + f"{testsuite.name()}: {testsuite.successes()}/{len(testsuite.test_cases)} successful" + ) + for testcase in testsuite.test_cases: + print(f" {testcase.client_types}: {testcase.state}") + if testcase.state != "success" and testcase.state != "waiting": + print(f"{testcase.exceptions}") return app diff --git a/trafficlight/http/adapter.py b/trafficlight/http/adapter.py index f8183fa..8626b90 100644 --- a/trafficlight/http/adapter.py +++ b/trafficlight/http/adapter.py @@ -132,6 +132,7 @@ async def loop_cleanup_unresponsive_adapters() -> None: sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) try: sleeping_tasks.add(sleep_task) + await sleep_task except asyncio.CancelledError: pass # we don't mind this task being cancelled. finally: @@ -146,6 +147,7 @@ async def loop_check_for_new_tests() -> None: sleep_task: asyncio.Future[None] = asyncio.ensure_future(asyncio.sleep(30)) try: sleeping_tasks.add(sleep_task) + await sleep_task except asyncio.CancelledError: pass # we don't mind this task being cancelled. finally: diff --git a/trafficlight/internals/testcase.py b/trafficlight/internals/testcase.py index 578073d..0b2311e 100644 --- a/trafficlight/internals/testcase.py +++ b/trafficlight/internals/testcase.py @@ -1,3 +1,4 @@ +import asyncio import hashlib import logging import traceback @@ -40,6 +41,7 @@ def __init__( self.servers: List[HomeServer] = [] self.files: Dict[str, str] = {} self.adapters: Optional[Dict[str, Adapter]] = None + self.completed = asyncio.get_event_loop().create_future() def __repr__(self) -> str: return f"{self.test.name()} ({self.server_type} {self.client_types})" @@ -111,7 +113,7 @@ async def run( # Treating an adapter that causes another type of exception as an error self.state = "error" self.exceptions.append(e.formatted_message) - except Exception: + except Exception as e: # Treating everything else as an error as well... eg compilation failures self.state = "error" self.exceptions.append("".join(traceback.format_exc())) @@ -122,3 +124,4 @@ async def run( server.finished() if kiwi.kiwi_client: await kiwi.kiwi_client.report_status(self) + self.completed.set_result(True) diff --git a/trafficlight/internals/testsuite.py b/trafficlight/internals/testsuite.py index 5d11cde..7589710 100644 --- a/trafficlight/internals/testsuite.py +++ b/trafficlight/internals/testsuite.py @@ -1,3 +1,4 @@ +import asyncio import hashlib from typing import List @@ -10,6 +11,8 @@ def __init__(self, test: Test, test_cases: List[TestCase]): self.guid = hashlib.md5(f"TestSuite{test.name()}".encode("utf-8")).hexdigest() self.test = test self.test_cases = test_cases + futures = list(map(lambda c: c.completed, test_cases)) + self.completed = asyncio.gather(*futures) def name(self) -> str: return self.test.name()