Support cpp stack traces

web/.flake8

@@ -2,6 +2,6 @@
 
 exclude =
     services/migrations,
-    services/github_issue_manager/test_search_for_matching_stacktrace.py,
-    services/github_issue_manager/test_trim_stacktrace.py,
+    services/utils/test_search_for_matching_stacktrace.py,
+    services/utils/test_trim_stacktrace.py,
 

web/services/migrations/0009_errorreport_cppcompressedtraces.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-11-29 13:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('services', '0008_add_github_issue_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='errorreport',
+            name='cppCompressedTraces',
+            field=models.CharField(blank=True, default='', max_length=10000),
+        ),
+    ]

web/services/migrations/0010_alter_errorreport_mantidversion.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-12-05 10:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('services', '0009_errorreport_cppcompressedtraces'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='errorreport',
+            name='mantidVersion',
+            field=models.CharField(max_length=64),
+        ),
+    ]

web/services/models.py

@@ -4,7 +4,12 @@
 from django.db.models import signals
 from services.tasks import send_notification_to_slack
 from services.constants import input_box_max_length, free_text_max_length
+from services.utils.handel_compressed_cpp_traces import (
+    extract_mantid_code_threads_from_cpp_traces
+)
 import threading
+
+
 # Implements saving recovery files to disk
 FILE_SYSTEM_STORE = FileSystemStorage(location=settings.MEDIA_ROOT)
 
@@ -24,7 +29,7 @@ class ErrorReport(models.Model):
     # ex: "3.17.4-200.fc20.x86_64"
     osVersion = models.CharField(max_length=32)
     ParaView = models.CharField(max_length=16)  # ex: "3.98.1"
-    mantidVersion = models.CharField(max_length=32)  # ex: "3.2.20141208.1820"
+    mantidVersion = models.CharField(max_length=64)  # ex: "3.2.20141208.1820"
     # sha1 ex: "e9423bdb34b07213a69caa90913e40307c17c6cc"
     mantidSha1 = models.CharField(max_length=40,
                                   help_text="sha1 for specific mantid version")
@@ -46,6 +51,8 @@ class ErrorReport(models.Model):
                                default="",
                                null="True")
     stacktrace = models.CharField(max_length=10000, default="")
+    cppCompressedTraces = models.CharField(max_length=10000, default="",
+                                           blank=True)
     githubIssue = models.ForeignKey('GithubIssue',
                                     on_delete=models.SET_NULL,
                                     blank=True,
@@ -99,6 +106,10 @@ def notify_report_received(sender, instance, signal, *args, **kwargs):
     textBox = instance.textBox
     stacktrace = instance.stacktrace
 
+    if instance.cppCompressedTraces != "":
+        stacktrace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(
+                                 instance.cppCompressedTraces))
+
     if instance.user is None:
 
         if ((stacktrace in TEST_VALUES
@@ -120,6 +131,7 @@ def notify_report_received(sender, instance, signal, *args, **kwargs):
          and textBox in TEST_VALUES)):
         return
 
+    issue_link = ""
     if instance.githubIssue:
         issue_link = (f"https://github.com/{instance.githubIssue.repoName}"
                       f"/issues/{instance.githubIssue.issueNumber}")
@@ -128,7 +140,7 @@ def notify_report_received(sender, instance, signal, *args, **kwargs):
         target=send_notification_to_slack, args=(name,
                                                  email,
                                                  instance.textBox,
-                                                 instance.stacktrace,
+                                                 stacktrace,
                                                  instance.application,
                                                  instance.mantidVersion,
                                                  instance.osReadable,

web/services/serializer.py

@@ -13,4 +13,5 @@ class Meta:
         fields = ['osReadable', 'application', 'url', 'uid',
                   'host', 'dateTime', 'osName', 'osArch', 'osVersion',
                   'ParaView', 'mantidVersion', 'mantidSha1', 'facility',
-                  'exitCode', 'upTime']
+                  'exitCode', 'upTime', 'textBox', 'stacktrace',
+                  'cppCompressedTraces', 'name', 'email']

web/services/tasks.py

@@ -8,10 +8,9 @@
 Name: $name Email: $email
 Additional text:
 $add_text
-Stack Trace:
-$stacktrace
 Using: $application $version on $os
-$issue_link
+Issue link: $issue_link
+Stack Trace:
 """)
 
 
@@ -37,20 +36,27 @@ def send_notification_to_slack(name,
         return
     text = SLACK_MESSAGE.substitute(
         name=_string_or_empty_field(name),
-        email=_string_or_empty_field(name),
+        email=_string_or_empty_field(email),
         add_text=_string_or_empty_field(additional_text),
-        stacktrace=_string_or_empty_field(stacktrace),
         application=_string_or_empty_field(application),
         version=_string_or_empty_field(version),
         os=_string_or_empty_field(os),
-        issue_link=_string_or_empty_field(github_issue_link)
+        issue_link=github_issue_link
     )
+    stacktrace_text = f"```{_string_or_empty_field(stacktrace)}```"
     requests.post(slack_webhook_url,
                   json={
                       'channel': settings.SLACK_ERROR_REPORTS_CHANNEL,
                       'username': settings.SLACK_ERROR_REPORTS_USERNAME,
                       'text': text,
-                      'icon_emoji': settings.SLACK_ERROR_REPORTS_EMOJI
+                      'icon_emoji': settings.SLACK_ERROR_REPORTS_EMOJI,
+                      'attachments':
+                      [
+                          {
+                              'mrkdwn_in': ['text'],
+                              'text': stacktrace_text
+                          }
+                      ]
                   })
 
 

web/services/github_issue_manager/github_issue_manager.py → web/services/utils/github_issue_manager.py

@@ -1,4 +1,7 @@
 from services.models import ErrorReport, GithubIssue
+from services.utils.handel_compressed_cpp_traces import (
+    extract_mantid_code_threads_from_cpp_traces
+)
 
 import re
 import pathlib
@@ -58,7 +61,10 @@ def get_or_create_github_issue(report) -> GithubIssue | None:
         GithubIssue | None: A reference to a new or existing GithubIssue table
         entry, or None
     """
-    if not report.get('stacktrace') and not report.get('textBox'):
+    stacktrace = report.get('stacktrace')
+    text_box = report.get('textBox')
+    cpp_compressed_traces = report.get('cppCompressedTraces')
+    if not any([stacktrace, text_box, cpp_compressed_traces]):
         logger.info('No stacktrace or info in the report; skipping github'
                     ' issue interaction')
         return None
@@ -74,7 +80,7 @@ def get_or_create_github_issue(report) -> GithubIssue | None:
     g = Github(auth=auth)
     repo = g.get_repo(issue_repo)
 
-    github_issue = _search_for_matching_stacktrace(report["stacktrace"])
+    github_issue = _search_for_matching_stacktrace(stacktrace)
     if github_issue and issue_repo == github_issue.repoName:
         issue_number = github_issue.issueNumber
         if (_search_for_repeat_user(report['uid'], github_issue) and
@@ -93,13 +99,18 @@ def get_or_create_github_issue(report) -> GithubIssue | None:
         logger.info(f'Added comment to issue {issue.url})')
         return github_issue
     else:
+        trace = stacktrace
+        if cpp_compressed_traces:
+            trace = "\n\n".join(extract_mantid_code_threads_from_cpp_traces(
+                                cpp_compressed_traces))
+
         issue_text = ISSUE_TEXT.substitute(
             name=report['name'],
             email=report['email'],
             os=report['osReadable'],
             version=report['mantidVersion'],
             info=report['textBox'],
-            stacktrace=report['stacktrace']
+            stacktrace=trace
         )
         error_report_label = repo.get_label("Error Report")
         issue = repo.create_issue(title="Automatic error report",

web/services/utils/handel_compressed_cpp_traces.py

@@ -0,0 +1,33 @@
+import base64
+import re
+import zlib
+
+
+def extract_mantid_code_threads_from_cpp_traces(compressed_cpp_traces: str):
+    """
+    Take base64 encoded string of the compressed output from pystack core.
+    Return a list of trace back threads which includes code from
+    the mantid repo.
+    """
+    cpp_traces_from_pystack = zlib.decompress(
+        base64.standard_b64decode(compressed_cpp_traces)).decode("utf-8")
+    traces = re.split(r'\nTraceback for ', cpp_traces_from_pystack)
+    if len(traces > 1):
+        # Trim boilerplate output
+        traces = traces[1:]
+    return ["Traceback for " + trace_back for trace_back in
+            traces if _search_for_mantid_codein_trace(trace_back)]
+
+
+def _search_for_mantid_codein_trace(trace_back: str) -> bool:
+    cpp_mantid_code = re.search(
+        r"^\s*\(C\) File \".*/(mantid|mantidqt|mantidqtinterfaces|workbench|"
+        r"scripts|plugins)/.*$",
+        trace_back,
+        re.MULTILINE) is not None
+    python_mantid_code = re.search(
+        r"^\s*\(Python\) File \".*/(mantid|"
+        r"mantidqt|mantidqtinterfaces|workbench|scripts|plugins)/.*$",
+        trace_back,
+        re.MULTILINE) is not None
+    return cpp_mantid_code or python_mantid_code

web/services/github_issue_manager/test_search_for_matching_stacktrace.py → web/services/utils/test_search_for_matching_stacktrace.py

@@ -1,6 +1,6 @@
 from django.test import TestCase
 from services.models import ErrorReport, GithubIssue
-from services.github_issue_manager.github_issue_manager import _search_for_matching_stacktrace
+from services.utils.github_issue_manager import _search_for_matching_stacktrace
 
 
 class MatchingStackTraceSearchTest(TestCase):

web/services/github_issue_manager/test_trim_stacktrace.py → web/services/utils/test_trim_stacktrace.py

@@ -1,4 +1,4 @@
-from services.github_issue_manager.github_issue_manager import _trim_stacktrace, _stacktrace_line_trimer
+from services.utils.github_issue_manager import _trim_stacktrace, _stacktrace_line_trimer
 import unittest
 
 

web/services/views.py

@@ -1,5 +1,5 @@
 from services.models import ErrorReport, UserDetails
-from services.github_issue_manager.github_issue_manager import (
+from services.utils.github_issue_manager import (
     get_or_create_github_issue
 )
 from services.constants import input_box_max_length
@@ -102,6 +102,8 @@ def saveErrorReport(report):
     exitCode = report["exitCode"]
     textBox = report["textBox"] if "textBox" in report else ""
     stacktrace = report["stacktrace"] if "stacktrace" in report else ""
+    cppCompressedTraces = report["cppCompressedTraces"] \
+        if "cppCompressedTraces" in report else ""
 
     if "name" in report and "email" in report:
         name = report["name"]
@@ -119,24 +121,26 @@ def saveErrorReport(report):
 
     github_issue = get_or_create_github_issue(report)
 
-    obj, created = \
-        ErrorReport.objects.get_or_create(osReadable=osReadable,
-                                          application=application,
-                                          uid=uid, host=host,
-                                          dateTime=dateTime,
-                                          osName=osName,
-                                          osArch=osArch,
-                                          osVersion=osVersion,
-                                          ParaView=ParaView,
-                                          mantidVersion=mantidVersion,
-                                          mantidSha1=mantidSha1,
-                                          facility=facility,
-                                          upTime=upTime,
-                                          exitCode=exitCode,
-                                          user=user,
-                                          textBox=textBox,
-                                          stacktrace=stacktrace,
-                                          githubIssue=github_issue)
+    obj, created = ErrorReport.objects.get_or_create(
+        osReadable=osReadable,
+        application=application,
+        uid=uid, host=host,
+        dateTime=dateTime,
+        osName=osName,
+        osArch=osArch,
+        osVersion=osVersion,
+        ParaView=ParaView,
+        mantidVersion=mantidVersion,
+        mantidSha1=mantidSha1,
+        facility=facility,
+        upTime=upTime,
+        exitCode=exitCode,
+        user=user,
+        textBox=textBox,
+        stacktrace=stacktrace,
+        cppCompressedTraces=cppCompressedTraces,
+        githubIssue=github_issue
+    )
     if not created:
         obj.save()