From 6f5629b160d942650a4e22ed1004e3a72a6b4fcb Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Thu, 5 Dec 2024 10:39:14 +0000 Subject: [PATCH 1/8] Add new model field cppCompressedTraces cppCompressedTRaces will be stoed as a base64 compressed string to avoid complications I was having with sending compressed bytes to django. Had to up the max length on mantidVersion, my dev build with '+uncommited' was reaching 36 characters. --- .../0009_errorreport_cppcompressedtraces.py | 18 +++++++++++++ .../0010_alter_errorreport_mantidversion.py | 18 +++++++++++++ web/services/models.py | 27 +++++++++++++++++-- web/services/serializer.py | 3 ++- web/services/views.py | 2 ++ 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 web/services/migrations/0009_errorreport_cppcompressedtraces.py create mode 100644 web/services/migrations/0010_alter_errorreport_mantidversion.py diff --git a/web/services/migrations/0009_errorreport_cppcompressedtraces.py b/web/services/migrations/0009_errorreport_cppcompressedtraces.py new file mode 100644 index 0000000..ce8937f --- /dev/null +++ b/web/services/migrations/0009_errorreport_cppcompressedtraces.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-11-29 13:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('services', '0008_add_github_issue_model'), + ] + + operations = [ + migrations.AddField( + model_name='errorreport', + name='cppCompressedTraces', + field=models.CharField(blank=True, default='', max_length=10000), + ), + ] diff --git a/web/services/migrations/0010_alter_errorreport_mantidversion.py b/web/services/migrations/0010_alter_errorreport_mantidversion.py new file mode 100644 index 0000000..976e151 --- /dev/null +++ b/web/services/migrations/0010_alter_errorreport_mantidversion.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-12-05 10:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('services', '0009_errorreport_cppcompressedtraces'), + ] + + operations = [ + migrations.AlterField( + model_name='errorreport', + name='mantidVersion', + field=models.CharField(max_length=64), + ), + ] \ No newline at end of file diff --git a/web/services/models.py b/web/services/models.py index 62c140f..55326ee 100644 --- a/web/services/models.py +++ b/web/services/models.py @@ -4,7 +4,13 @@ from django.db.models import signals from services.tasks import send_notification_to_slack from services.constants import input_box_max_length, free_text_max_length +import base64 +import logging +import re import threading +import zlib + +logger = logging.getLogger("Error report model") # Implements saving recovery files to disk FILE_SYSTEM_STORE = FileSystemStorage(location=settings.MEDIA_ROOT) @@ -24,7 +30,7 @@ class ErrorReport(models.Model): # ex: "3.17.4-200.fc20.x86_64" osVersion = models.CharField(max_length=32) ParaView = models.CharField(max_length=16) # ex: "3.98.1" - mantidVersion = models.CharField(max_length=32) # ex: "3.2.20141208.1820" + mantidVersion = models.CharField(max_length=64) # ex: "3.2.20141208.1820" # sha1 ex: "e9423bdb34b07213a69caa90913e40307c17c6cc" mantidSha1 = models.CharField(max_length=40, help_text="sha1 for specific mantid version") @@ -46,6 +52,7 @@ class ErrorReport(models.Model): default="", null="True") stacktrace = models.CharField(max_length=10000, default="") + cppCompressedTraces = models.CharField(max_length=10000, default="", blank=True) githubIssue = models.ForeignKey('GithubIssue', on_delete=models.SET_NULL, blank=True, @@ -86,6 +93,18 @@ class GithubIssue(models.Model): issueNumber = models.CharField(max_length=16, default="", blank=True) +def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str): + cpp_traces_from_pystack = zlib.decompress(base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") + return ["Traceback for " + trace_back for trace_back in re.split(r'\n\nTraceback for ', cpp_traces_from_pystack) if + search_for_mantid_codein_trace(trace_back)] + + +def search_for_mantid_codein_trace(trace_back: str) -> bool: + cpp_mantid_code = re.search(r"^\s*\(C\) File \".*(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins).*$", trace_back, re.MULTILINE) is not None + python_mantid_code = re.search(r"^\s*\(Python\) File \".*(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins).*$", trace_back, re.MULTILINE) is not None + return cpp_mantid_code or python_mantid_code + + def notify_report_received(sender, instance, signal, *args, **kwargs): """ Send a notification to the defined endpoint when a new error @@ -99,6 +118,9 @@ def notify_report_received(sender, instance, signal, *args, **kwargs): textBox = instance.textBox stacktrace = instance.stacktrace + if instance.cppCompressedTraces != "": + stacktrace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(instance.cppCompressedTraces)) + if instance.user is None: if ((stacktrace in TEST_VALUES @@ -120,6 +142,7 @@ def notify_report_received(sender, instance, signal, *args, **kwargs): and textBox in TEST_VALUES)): return + issue_link = "" if instance.githubIssue: issue_link = (f"https://github.com/{instance.githubIssue.repoName}" f"/issues/{instance.githubIssue.issueNumber}") @@ -128,7 +151,7 @@ def notify_report_received(sender, instance, signal, *args, **kwargs): target=send_notification_to_slack, args=(name, email, instance.textBox, - instance.stacktrace, + stacktrace, instance.application, instance.mantidVersion, instance.osReadable, diff --git a/web/services/serializer.py b/web/services/serializer.py index e65b9bb..5866ca3 100644 --- a/web/services/serializer.py +++ b/web/services/serializer.py @@ -13,4 +13,5 @@ class Meta: fields = ['osReadable', 'application', 'url', 'uid', 'host', 'dateTime', 'osName', 'osArch', 'osVersion', 'ParaView', 'mantidVersion', 'mantidSha1', 'facility', - 'exitCode', 'upTime'] + 'exitCode', 'upTime', 'textBox', 'stacktrace', 'cppCompressedTraces' + 'name', 'email'] diff --git a/web/services/views.py b/web/services/views.py index 9c33e4a..4634f21 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -102,6 +102,7 @@ def saveErrorReport(report): exitCode = report["exitCode"] textBox = report["textBox"] if "textBox" in report else "" stacktrace = report["stacktrace"] if "stacktrace" in report else "" + cppCompressedTraces = report["cppCompressedTraces"] if "cppCompressedTraces" in report else "" if "name" in report and "email" in report: name = report["name"] @@ -136,6 +137,7 @@ def saveErrorReport(report): user=user, textBox=textBox, stacktrace=stacktrace, + cppCompressedTraces=cppCompressedTraces, githubIssue=github_issue) if not created: obj.save() From 31c6da5b8118dba61af3a240f49a79e10b8d0425 Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Thu, 5 Dec 2024 16:07:08 +0000 Subject: [PATCH 2/8] imporve trimmed down formatting --- web/services/models.py | 6 +++--- web/services/tasks.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/services/models.py b/web/services/models.py index 55326ee..267aa38 100644 --- a/web/services/models.py +++ b/web/services/models.py @@ -95,13 +95,13 @@ class GithubIssue(models.Model): def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str): cpp_traces_from_pystack = zlib.decompress(base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") - return ["Traceback for " + trace_back for trace_back in re.split(r'\n\nTraceback for ', cpp_traces_from_pystack) if + return ["Traceback for " + trace_back for trace_back in re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if search_for_mantid_codein_trace(trace_back)] def search_for_mantid_codein_trace(trace_back: str) -> bool: - cpp_mantid_code = re.search(r"^\s*\(C\) File \".*(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins).*$", trace_back, re.MULTILINE) is not None - python_mantid_code = re.search(r"^\s*\(Python\) File \".*(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins).*$", trace_back, re.MULTILINE) is not None + cpp_mantid_code = re.search(r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None + python_mantid_code = re.search(r"^\s*\(Python\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None return cpp_mantid_code or python_mantid_code diff --git a/web/services/tasks.py b/web/services/tasks.py index f415ba5..845b8fa 100644 --- a/web/services/tasks.py +++ b/web/services/tasks.py @@ -9,7 +9,7 @@ Additional text: $add_text Stack Trace: -$stacktrace +```$stacktrace``` Using: $application $version on $os $issue_link """) From 210c195765d564756c5d4b4d5dbb38870daba8fb Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Mon, 9 Dec 2024 14:26:51 +0000 Subject: [PATCH 3/8] move cpp trace code to utils module and add to github issue maker --- web/services/models.py | 17 ++-------------- .../__init__.py | 0 .../github_issue_manager.py | 14 ++++++++++--- .../utils/handel_compressed_cpp_traces.py | 20 +++++++++++++++++++ .../test_search_for_matching_stacktrace.py | 2 +- .../test_trim_stacktrace.py | 2 +- web/services/views.py | 2 +- 7 files changed, 36 insertions(+), 21 deletions(-) rename web/services/{github_issue_manager => utils}/__init__.py (100%) rename web/services/{github_issue_manager => utils}/github_issue_manager.py (90%) create mode 100644 web/services/utils/handel_compressed_cpp_traces.py rename web/services/{github_issue_manager => utils}/test_search_for_matching_stacktrace.py (97%) rename web/services/{github_issue_manager => utils}/test_trim_stacktrace.py (95%) diff --git a/web/services/models.py b/web/services/models.py index 267aa38..17b0b43 100644 --- a/web/services/models.py +++ b/web/services/models.py @@ -4,11 +4,10 @@ from django.db.models import signals from services.tasks import send_notification_to_slack from services.constants import input_box_max_length, free_text_max_length -import base64 +from services.utils.handel_compressed_cpp_traces import extract_mantid_code_threads_from_cpp_traces import logging -import re import threading -import zlib + logger = logging.getLogger("Error report model") # Implements saving recovery files to disk @@ -93,18 +92,6 @@ class GithubIssue(models.Model): issueNumber = models.CharField(max_length=16, default="", blank=True) -def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str): - cpp_traces_from_pystack = zlib.decompress(base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") - return ["Traceback for " + trace_back for trace_back in re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if - search_for_mantid_codein_trace(trace_back)] - - -def search_for_mantid_codein_trace(trace_back: str) -> bool: - cpp_mantid_code = re.search(r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None - python_mantid_code = re.search(r"^\s*\(Python\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None - return cpp_mantid_code or python_mantid_code - - def notify_report_received(sender, instance, signal, *args, **kwargs): """ Send a notification to the defined endpoint when a new error diff --git a/web/services/github_issue_manager/__init__.py b/web/services/utils/__init__.py similarity index 100% rename from web/services/github_issue_manager/__init__.py rename to web/services/utils/__init__.py diff --git a/web/services/github_issue_manager/github_issue_manager.py b/web/services/utils/github_issue_manager.py similarity index 90% rename from web/services/github_issue_manager/github_issue_manager.py rename to web/services/utils/github_issue_manager.py index 9142a96..ed7b405 100644 --- a/web/services/github_issue_manager/github_issue_manager.py +++ b/web/services/utils/github_issue_manager.py @@ -1,4 +1,5 @@ from services.models import ErrorReport, GithubIssue +from services.utils.handel_compressed_cpp_traces import extract_mantid_code_threads_from_cpp_traces import re import pathlib @@ -58,7 +59,10 @@ def get_or_create_github_issue(report) -> GithubIssue | None: GithubIssue | None: A reference to a new or existing GithubIssue table entry, or None """ - if not report.get('stacktrace') and not report.get('textBox'): + stacktrace = report.get('stacktrace') + text_box = report.get('textBox') + cpp_compressed_traces = report.get('cppCompressedTraces') + if stacktrace is None and text_box is None and cpp_compressed_traces is None: logger.info('No stacktrace or info in the report; skipping github' ' issue interaction') return None @@ -74,7 +78,7 @@ def get_or_create_github_issue(report) -> GithubIssue | None: g = Github(auth=auth) repo = g.get_repo(issue_repo) - github_issue = _search_for_matching_stacktrace(report["stacktrace"]) + github_issue = _search_for_matching_stacktrace(stacktrace) if github_issue and issue_repo == github_issue.repoName: issue_number = github_issue.issueNumber if (_search_for_repeat_user(report['uid'], github_issue) and @@ -93,13 +97,17 @@ def get_or_create_github_issue(report) -> GithubIssue | None: logger.info(f'Added comment to issue {issue.url})') return github_issue else: + trace = stacktrace + if cpp_compressed_traces: + trace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(cpp_compressed_traces)) + issue_text = ISSUE_TEXT.substitute( name=report['name'], email=report['email'], os=report['osReadable'], version=report['mantidVersion'], info=report['textBox'], - stacktrace=report['stacktrace'] + stacktrace=trace ) error_report_label = repo.get_label("Error Report") issue = repo.create_issue(title="Automatic error report", diff --git a/web/services/utils/handel_compressed_cpp_traces.py b/web/services/utils/handel_compressed_cpp_traces.py new file mode 100644 index 0000000..ee18013 --- /dev/null +++ b/web/services/utils/handel_compressed_cpp_traces.py @@ -0,0 +1,20 @@ +import base64 +import re +from typing import List +import zlib + + +def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str) -> List[str]: + """ + Take base64 encoded string of the compressed output from pystack core. + Return a list of trace back threads which include code from the mantid repo. + """ + cpp_traces_from_pystack = zlib.decompress(base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") + return ["Traceback for " + trace_back for trace_back in re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if + _search_for_mantid_codein_trace(trace_back)] + + +def _search_for_mantid_codein_trace(trace_back: str) -> bool: + cpp_mantid_code = re.search(r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None + python_mantid_code = re.search(r"^\s*\(Python\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None + return cpp_mantid_code or python_mantid_code \ No newline at end of file diff --git a/web/services/github_issue_manager/test_search_for_matching_stacktrace.py b/web/services/utils/test_search_for_matching_stacktrace.py similarity index 97% rename from web/services/github_issue_manager/test_search_for_matching_stacktrace.py rename to web/services/utils/test_search_for_matching_stacktrace.py index fff81fd..1f46c9f 100644 --- a/web/services/github_issue_manager/test_search_for_matching_stacktrace.py +++ b/web/services/utils/test_search_for_matching_stacktrace.py @@ -1,6 +1,6 @@ from django.test import TestCase from services.models import ErrorReport, GithubIssue -from services.github_issue_manager.github_issue_manager import _search_for_matching_stacktrace +from services.utils.github_issue_manager import _search_for_matching_stacktrace class MatchingStackTraceSearchTest(TestCase): diff --git a/web/services/github_issue_manager/test_trim_stacktrace.py b/web/services/utils/test_trim_stacktrace.py similarity index 95% rename from web/services/github_issue_manager/test_trim_stacktrace.py rename to web/services/utils/test_trim_stacktrace.py index 9add7a0..d9196f3 100644 --- a/web/services/github_issue_manager/test_trim_stacktrace.py +++ b/web/services/utils/test_trim_stacktrace.py @@ -1,4 +1,4 @@ -from services.github_issue_manager.github_issue_manager import _trim_stacktrace, _stacktrace_line_trimer +from services.utils.github_issue_manager import _trim_stacktrace, _stacktrace_line_trimer import unittest diff --git a/web/services/views.py b/web/services/views.py index 4634f21..27451b0 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -1,5 +1,5 @@ from services.models import ErrorReport, UserDetails -from services.github_issue_manager.github_issue_manager import ( +from services.utils.github_issue_manager import ( get_or_create_github_issue ) from services.constants import input_box_max_length From 4086aada0a8459a5ee4469df5fe3765df0f808ad Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Tue, 10 Dec 2024 14:58:57 +0000 Subject: [PATCH 4/8] flake8 changes --- web/.flake8 | 4 +- web/services/models.py | 10 +++-- web/services/serializer.py | 4 +- web/services/utils/github_issue_manager.py | 9 ++-- .../utils/handel_compressed_cpp_traces.py | 26 ++++++++---- web/services/views.py | 42 ++++++++++--------- 6 files changed, 57 insertions(+), 38 deletions(-) diff --git a/web/.flake8 b/web/.flake8 index 1c97031..45ccc9a 100644 --- a/web/.flake8 +++ b/web/.flake8 @@ -2,6 +2,6 @@ exclude = services/migrations, - services/github_issue_manager/test_search_for_matching_stacktrace.py, - services/github_issue_manager/test_trim_stacktrace.py, + services/utils/test_search_for_matching_stacktrace.py, + services/utils/test_trim_stacktrace.py, diff --git a/web/services/models.py b/web/services/models.py index 17b0b43..6e1c27d 100644 --- a/web/services/models.py +++ b/web/services/models.py @@ -4,7 +4,9 @@ from django.db.models import signals from services.tasks import send_notification_to_slack from services.constants import input_box_max_length, free_text_max_length -from services.utils.handel_compressed_cpp_traces import extract_mantid_code_threads_from_cpp_traces +from services.utils.handel_compressed_cpp_traces import ( + extract_mantid_code_threads_from_cpp_traces +) import logging import threading @@ -51,7 +53,8 @@ class ErrorReport(models.Model): default="", null="True") stacktrace = models.CharField(max_length=10000, default="") - cppCompressedTraces = models.CharField(max_length=10000, default="", blank=True) + cppCompressedTraces = models.CharField(max_length=10000, default="", + blank=True) githubIssue = models.ForeignKey('GithubIssue', on_delete=models.SET_NULL, blank=True, @@ -106,7 +109,8 @@ def notify_report_received(sender, instance, signal, *args, **kwargs): stacktrace = instance.stacktrace if instance.cppCompressedTraces != "": - stacktrace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(instance.cppCompressedTraces)) + stacktrace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces( + instance.cppCompressedTraces)) if instance.user is None: diff --git a/web/services/serializer.py b/web/services/serializer.py index 5866ca3..f8920c6 100644 --- a/web/services/serializer.py +++ b/web/services/serializer.py @@ -13,5 +13,5 @@ class Meta: fields = ['osReadable', 'application', 'url', 'uid', 'host', 'dateTime', 'osName', 'osArch', 'osVersion', 'ParaView', 'mantidVersion', 'mantidSha1', 'facility', - 'exitCode', 'upTime', 'textBox', 'stacktrace', 'cppCompressedTraces' - 'name', 'email'] + 'exitCode', 'upTime', 'textBox', 'stacktrace', + 'cppCompressedTraces', 'name', 'email'] diff --git a/web/services/utils/github_issue_manager.py b/web/services/utils/github_issue_manager.py index ed7b405..01e5dae 100644 --- a/web/services/utils/github_issue_manager.py +++ b/web/services/utils/github_issue_manager.py @@ -1,5 +1,7 @@ from services.models import ErrorReport, GithubIssue -from services.utils.handel_compressed_cpp_traces import extract_mantid_code_threads_from_cpp_traces +from services.utils.handel_compressed_cpp_traces import ( + extract_mantid_code_threads_from_cpp_traces +) import re import pathlib @@ -62,7 +64,7 @@ def get_or_create_github_issue(report) -> GithubIssue | None: stacktrace = report.get('stacktrace') text_box = report.get('textBox') cpp_compressed_traces = report.get('cppCompressedTraces') - if stacktrace is None and text_box is None and cpp_compressed_traces is None: + if not any([stacktrace, text_box, cpp_compressed_traces]): logger.info('No stacktrace or info in the report; skipping github' ' issue interaction') return None @@ -99,7 +101,8 @@ def get_or_create_github_issue(report) -> GithubIssue | None: else: trace = stacktrace if cpp_compressed_traces: - trace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(cpp_compressed_traces)) + trace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces( + cpp_compressed_traces)) issue_text = ISSUE_TEXT.substitute( name=report['name'], diff --git a/web/services/utils/handel_compressed_cpp_traces.py b/web/services/utils/handel_compressed_cpp_traces.py index ee18013..c3c601b 100644 --- a/web/services/utils/handel_compressed_cpp_traces.py +++ b/web/services/utils/handel_compressed_cpp_traces.py @@ -1,20 +1,30 @@ import base64 import re -from typing import List import zlib -def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str) -> List[str]: +def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str): """ Take base64 encoded string of the compressed output from pystack core. - Return a list of trace back threads which include code from the mantid repo. + Return a list of trace back threads which includes code from + the mantid repo. """ - cpp_traces_from_pystack = zlib.decompress(base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") - return ["Traceback for " + trace_back for trace_back in re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if + cpp_traces_from_pystack = zlib.decompress( + base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") + return ["Traceback for " + trace_back for trace_back in + re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if _search_for_mantid_codein_trace(trace_back)] def _search_for_mantid_codein_trace(trace_back: str) -> bool: - cpp_mantid_code = re.search(r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None - python_mantid_code = re.search(r"^\s*\(Python\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", trace_back, re.MULTILINE) is not None - return cpp_mantid_code or python_mantid_code \ No newline at end of file + cpp_mantid_code = re.search( + r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|" + r"scripts|plugins)/.*$", + trace_back, + re.MULTILINE) is not None + python_mantid_code = re.search( + r"^\s*\(Python\) File \".*/(mantid|" + r"mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$", + trace_back, + re.MULTILINE) is not None + return cpp_mantid_code or python_mantid_code diff --git a/web/services/views.py b/web/services/views.py index 27451b0..dd29d1c 100644 --- a/web/services/views.py +++ b/web/services/views.py @@ -102,7 +102,8 @@ def saveErrorReport(report): exitCode = report["exitCode"] textBox = report["textBox"] if "textBox" in report else "" stacktrace = report["stacktrace"] if "stacktrace" in report else "" - cppCompressedTraces = report["cppCompressedTraces"] if "cppCompressedTraces" in report else "" + cppCompressedTraces = report["cppCompressedTraces"] \ + if "cppCompressedTraces" in report else "" if "name" in report and "email" in report: name = report["name"] @@ -120,25 +121,26 @@ def saveErrorReport(report): github_issue = get_or_create_github_issue(report) - obj, created = \ - ErrorReport.objects.get_or_create(osReadable=osReadable, - application=application, - uid=uid, host=host, - dateTime=dateTime, - osName=osName, - osArch=osArch, - osVersion=osVersion, - ParaView=ParaView, - mantidVersion=mantidVersion, - mantidSha1=mantidSha1, - facility=facility, - upTime=upTime, - exitCode=exitCode, - user=user, - textBox=textBox, - stacktrace=stacktrace, - cppCompressedTraces=cppCompressedTraces, - githubIssue=github_issue) + obj, created = ErrorReport.objects.get_or_create( + osReadable=osReadable, + application=application, + uid=uid, host=host, + dateTime=dateTime, + osName=osName, + osArch=osArch, + osVersion=osVersion, + ParaView=ParaView, + mantidVersion=mantidVersion, + mantidSha1=mantidSha1, + facility=facility, + upTime=upTime, + exitCode=exitCode, + user=user, + textBox=textBox, + stacktrace=stacktrace, + cppCompressedTraces=cppCompressedTraces, + githubIssue=github_issue + ) if not created: obj.save() From 2e2a0697e5caa14ab164a6a30cb1e4e9e921d5e2 Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Wed, 8 Jan 2025 15:50:29 +0000 Subject: [PATCH 5/8] fix name being used as email --- web/services/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/services/tasks.py b/web/services/tasks.py index 845b8fa..ac9494f 100644 --- a/web/services/tasks.py +++ b/web/services/tasks.py @@ -37,7 +37,7 @@ def send_notification_to_slack(name, return text = SLACK_MESSAGE.substitute( name=_string_or_empty_field(name), - email=_string_or_empty_field(name), + email=_string_or_empty_field(email), add_text=_string_or_empty_field(additional_text), stacktrace=_string_or_empty_field(stacktrace), application=_string_or_empty_field(application), From d800f850aba429ac3296f8fde6649c7bad7e324e Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Wed, 8 Jan 2025 15:51:18 +0000 Subject: [PATCH 6/8] add stack trace as attachment text so it collapses --- web/services/tasks.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/web/services/tasks.py b/web/services/tasks.py index ac9494f..4aacbb0 100644 --- a/web/services/tasks.py +++ b/web/services/tasks.py @@ -8,10 +8,9 @@ Name: $name Email: $email Additional text: $add_text -Stack Trace: -```$stacktrace``` Using: $application $version on $os -$issue_link +Issue link: $issue_link +Stack Trace: """) @@ -39,18 +38,25 @@ def send_notification_to_slack(name, name=_string_or_empty_field(name), email=_string_or_empty_field(email), add_text=_string_or_empty_field(additional_text), - stacktrace=_string_or_empty_field(stacktrace), application=_string_or_empty_field(application), version=_string_or_empty_field(version), os=_string_or_empty_field(os), - issue_link=_string_or_empty_field(github_issue_link) + issue_link=github_issue_link ) + stacktrace_text = f"```{_string_or_empty_field(stacktrace)}```" requests.post(slack_webhook_url, json={ 'channel': settings.SLACK_ERROR_REPORTS_CHANNEL, 'username': settings.SLACK_ERROR_REPORTS_USERNAME, 'text': text, - 'icon_emoji': settings.SLACK_ERROR_REPORTS_EMOJI + 'icon_emoji': settings.SLACK_ERROR_REPORTS_EMOJI, + 'attachments': + [ + { + 'mrkdwn_in': ['text'], + 'text': stacktrace_text + } + ] }) From a05f83dd9633bf660ccbd8a0d8fdbda6f77b91c1 Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Fri, 10 Jan 2025 11:19:11 +0000 Subject: [PATCH 7/8] add check to avoid out of bounds error --- web/services/utils/handel_compressed_cpp_traces.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web/services/utils/handel_compressed_cpp_traces.py b/web/services/utils/handel_compressed_cpp_traces.py index c3c601b..1be23d4 100644 --- a/web/services/utils/handel_compressed_cpp_traces.py +++ b/web/services/utils/handel_compressed_cpp_traces.py @@ -11,9 +11,12 @@ def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str): """ cpp_traces_from_pystack = zlib.decompress( base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8") + traces = re.split(r'\nTraceback for ', cpp_traces_from_pystack) + if len(traces > 1): + # Trim boilerplate output + traces = traces[1:] return ["Traceback for " + trace_back for trace_back in - re.split(r'\nTraceback for ', cpp_traces_from_pystack)[1:] if - _search_for_mantid_codein_trace(trace_back)] + traces if _search_for_mantid_codein_trace(trace_back)] def _search_for_mantid_codein_trace(trace_back: str) -> bool: From fe62aa04b83d565e34a3891279899c7f50bb26d8 Mon Sep 17 00:00:00 2001 From: Jonathan Haigh Date: Fri, 10 Jan 2025 11:25:13 +0000 Subject: [PATCH 8/8] remove unused logger --- web/services/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/services/models.py b/web/services/models.py index 6e1c27d..a134ab6 100644 --- a/web/services/models.py +++ b/web/services/models.py @@ -7,11 +7,9 @@ from services.utils.handel_compressed_cpp_traces import ( extract_mantid_code_threads_from_cpp_traces ) -import logging import threading -logger = logging.getLogger("Error report model") # Implements saving recovery files to disk FILE_SYSTEM_STORE = FileSystemStorage(location=settings.MEDIA_ROOT)