From 6f22d14d41052b2227d883bfdd561a7c30f34260 Mon Sep 17 00:00:00 2001 From: Karel Srot Date: Mon, 9 Sep 2024 10:57:12 +0200 Subject: [PATCH] reportportal: add logfile size upload limit --- docs/releases.rst | 9 +++ tmt/schemas/report/reportportal.yaml | 6 ++ tmt/steps/report/reportportal.py | 84 +++++++++++++++++++++++++++- 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 182a85764b..a99d30d17a 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -24,6 +24,15 @@ discovered tests. This can be useful especially when fetching tests from remote repositories where the user does not have write access. +The ``tmt reportportal`` plugin has newly introduced size limit +for logs uploaded to ReportPortal because large logs decreases +ReportPortal UI usability. Default limit are 1 MB for a test +output and 50 kB for a traceback (error log). +Limits can be controlled using the newly introduced +``reportportal`` plugin options ``-log-size-limit`` and +``--traceback-size-limit`` or the respective environment +variables. + tmt-1.37.0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tmt/schemas/report/reportportal.yaml b/tmt/schemas/report/reportportal.yaml index fef9cc0a6e..26a47bcfd4 100644 --- a/tmt/schemas/report/reportportal.yaml +++ b/tmt/schemas/report/reportportal.yaml @@ -64,6 +64,12 @@ properties: api-version: type: string + log-size-limit: + type: integer + + traceback-size-limit: + type: integer + required: - how - project diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index 91d0c2f5d6..b6ff3905a4 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -15,6 +15,8 @@ from tmt._compat.typing import TypeAlias JSON: 'TypeAlias' = Any +DEFAULT_LOG_SIZE_LIMIT: int = 1024 * 1024 +DEFAULT_TRACEBACK_SIZE_LIMIT: int = 50 * 1024 def _flag_env_to_default(option: str, default: bool) -> bool: @@ -41,13 +43,56 @@ def _str_env_to_default(option: str, default: Optional[str]) -> Optional[str]: return str(os.getenv(env_var)) -def _filter_invalid_chars(data: str) -> str: +class LogFilterSettings: + size: int + is_traceback: bool + + def __init__(self, + size: int = DEFAULT_LOG_SIZE_LIMIT, + is_traceback: bool = False): + self.size = size + self.is_traceback = is_traceback + + +def _filter_invalid_chars(data: str, + settings: LogFilterSettings) -> str: return re.sub( '[^\u0020-\uD7FF\u0009\u000A\u000D\uE000-\uFFFD\U00010000-\U0010FFFF]+', '', data) +def _filter_log_per_size(data: str, + settings: LogFilterSettings) -> str: + size = len(data) + if settings.is_traceback: + variable = "TMT_PLUGIN_REPORT_REPORTPORTAL_LOG_SIZE_LIMIT" + option = "--traceback-size-limit" + else: + variable = "TMT_PLUGIN_REPORT_REPORTPORTAL_LOG_SIZE_LIMIT" + option = "--log-size-limit" + header = (f"WARNING: Uploaded log has been truncated because its size {size} bytes " + f"exceeds tmt reportportal plugin limit of {settings.size} bytes." + f"The limit is controlled with {option} plugin option or" + f"{variable} environment variable.\n\n") + if size > settings.size: + return f"{header}{data[:settings.size]}" + return data + + +_LOG_FILTERS = [ + _filter_log_per_size, + _filter_invalid_chars, + ] + + +def _filter_log(log: str, settings: Optional[LogFilterSettings] = None) -> str: + settings = settings or LogFilterSettings() + for log_filter in _LOG_FILTERS: + log = log_filter(log, settings=settings) + return log + + @dataclasses.dataclass class ReportReportPortalData(tmt.steps.report.ReportStepData): @@ -142,6 +187,28 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): (e.g. 'Idle'). 'To Investigate' is used by default. """) + log_size_limit: int = field( + option="--log-size-limit", + metavar="LOG_SIZE_LIMIT", + default=int( + _str_env_to_default('log_size_limit', str(DEFAULT_LOG_SIZE_LIMIT))), + help=f""" + Size limit in bytes for log upload to ReportPortal. + The default limit is {DEFAULT_LOG_SIZE_LIMIT} bytes + ({DEFAULT_LOG_SIZE_LIMIT / 1024 / 1024} MB). + """) + + traceback_size_limit: int = field( + option="--traceback-size-limit", + metavar="TRACEBACK_SIZE_LIMIT", + default=int( + _str_env_to_default('traceback_size_limit', str(DEFAULT_TRACEBACK_SIZE_LIMIT))), + help=f""" + Size limit in bytes for traceback log upload to ReportPortal. + The default limit is {DEFAULT_TRACEBACK_SIZE_LIMIT} bytes + ({DEFAULT_TRACEBACK_SIZE_LIMIT / 1024} kB). + """) + exclude_variables: str = field( option="--exclude-variables", metavar="PATTERN", @@ -595,10 +662,16 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: status = self.TMT_TO_RP_RESULT_STATUS[result.result] # Upload log + + message = _filter_log(log, + settings=LogFilterSettings( + size=self.data.log_size_limit + ) + ) response = self.rp_api_post( session=session, path="log/entry", - json={"message": _filter_invalid_chars(log), + json={"message": message, "itemUuid": item_uuid, "launchUuid": launch_uuid, "level": level, @@ -606,7 +679,12 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: # Write out failures if index == 0 and status == "FAILED": - message = _filter_invalid_chars(result.failures(log)) + message = _filter_log(result.failures(log), + settings=LogFilterSettings( + size=self.data.traceback_size_limit, + is_traceback=True + ) + ) response = self.rp_api_post( session=session, path="log/entry",