From 0cb113217b84577992a60b42deddb4deac7c4778 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:37:32 +0000 Subject: [PATCH] Generate emails and github/gitlab issue bodies from a template (#84) The driver for this is that I want to add a link to our internal rageshake-search tool to the body of reported issues. Rather than add more magic special-casing that, making the format of issue bodies more configurable seems a good thing. --- CHANGES.md | 9 +++ README.md | 5 ++ main.go | 49 +++++++++++++++- rageshake.sample.yaml | 6 +- submit.go | 114 ++++++++++++++++++++++---------------- submit_test.go | 89 +++++++++++++++++++++++++---- templates/README.md | 26 +++++++++ templates/email_body.tmpl | 9 +++ templates/issue_body.tmpl | 9 +++ 9 files changed, 255 insertions(+), 61 deletions(-) create mode 100644 templates/README.md create mode 100644 templates/email_body.tmpl create mode 100644 templates/issue_body.tmpl diff --git a/CHANGES.md b/CHANGES.md index f775d4e..2129e2b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +UNRELEASED +========== + +Features +-------- + +- Allow configuration of the body of created Github/Gitlab issues via a template in the configuration file. ([\#84](https://github.com/matrix-org/rageshake/issues/84)) + + 1.11.0 (2023-08-11) =================== diff --git a/README.md b/README.md index 10b1f61..5c64f64 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ Optional parameters: * `-listen
`: TCP network address to listen for HTTP requests on. Example: `:9110`. +## Issue template + +It is possible to override the templates used to construct emails, and Github and Gitlab issues. +See [templates/README.md](templates/README.md) for more information. + ## HTTP endpoints The following HTTP endpoints are exposed: diff --git a/main.go b/main.go index 287e5e6..ffe3636 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "net/http" "os" "strings" + "text/template" "time" "github.com/google/go-github/github" @@ -36,6 +37,17 @@ import ( "gopkg.in/yaml.v2" ) +import _ "embed" + +// DefaultIssueBodyTemplate is the default template used for `issue_body_template_file` in the config. +// +//go:embed templates/issue_body.tmpl +var DefaultIssueBodyTemplate string + +// DefaultEmailBodyTemplate is the default template used for `email_body_template_file` in the config. +// +//go:embed templates/email_body.tmpl +var DefaultEmailBodyTemplate string var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.") var bindAddr = flag.String("listen", ":9110", "The port to listen on.") @@ -63,6 +75,9 @@ type config struct { GitlabProjectLabels map[string][]string `yaml:"gitlab_project_labels"` GitlabIssueConfidential bool `yaml:"gitlab_issue_confidential"` + IssueBodyTemplateFile string `yaml:"issue_body_template_file"` + EmailBodyTemplateFile string `yaml:"email_body_template_file"` + SlackWebhookURL string `yaml:"slack_webhook_url"` EmailAddresses []string `yaml:"email_addresses"` @@ -158,7 +173,17 @@ func main() { log.Printf("Using %s/listing as public URI", apiPrefix) rand.Seed(time.Now().UnixNano()) - http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, appNameMap, cfg}) + http.Handle("/api/submit", &submitServer{ + issueTemplate: parseTemplate(DefaultIssueBodyTemplate, cfg.IssueBodyTemplateFile, "issue"), + emailTemplate: parseTemplate(DefaultEmailBodyTemplate, cfg.EmailBodyTemplateFile, "email"), + ghClient: ghClient, + glClient: glClient, + apiPrefix: apiPrefix, + slack: slack, + genericWebhookClient: genericWebhookClient, + allowedAppNameMap: appNameMap, + cfg: cfg, + }) // Make sure bugs directory exists _ = os.Mkdir("bugs", os.ModePerm) @@ -186,6 +211,28 @@ func main() { log.Fatal(http.ListenAndServe(*bindAddr, nil)) } +// parseTemplate parses a template file, with fallback to default. +// +// If `templateFilePath` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is +// used. +// +// The template text is then parsed into a template named `templateName`. +func parseTemplate(defaultTemplate string, templateFilePath string, templateName string) *template.Template { + templateText := defaultTemplate + if templateFilePath != "" { + issueTemplateBytes, err := os.ReadFile(templateFilePath) + if err != nil { + log.Fatalf("Unable to read template file `%s`: %s", templateFilePath, err) + } + templateText = string(issueTemplateBytes) + } + parsedTemplate, err := template.New(templateName).Parse(templateText) + if err != nil { + log.Fatalf("Invalid template file %s in config file: %s", templateFilePath, err) + } + return parsedTemplate +} + func configureAppNameMap(cfg *config) map[string]bool { if len(cfg.AllowedAppNames) == 0 { fmt.Println("Warning: allowed_app_names is empty. Accepting requests from all app names") diff --git a/rageshake.sample.yaml b/rageshake.sample.yaml index 283e812..204a919 100644 --- a/rageshake.sample.yaml +++ b/rageshake.sample.yaml @@ -55,8 +55,12 @@ smtp_server: localhost:25 smtp_username: myemailuser smtp_password: myemailpass - # a list of webhook URLs, (see docs/generic_webhook.md) generic_webhook_urls: - https://server.example.com/your-server/api - http://another-server.com/api + +# The paths of template files for the body of Github and Gitlab issues, and emails. +# See `templates/README.md` for more information. +issue_body_template_file: path/to/issue_body.tmpl +email_body_template_file: path/to/email_body.tmpl diff --git a/submit.go b/submit.go index 38a0ec8..ae68edb 100644 --- a/submit.go +++ b/submit.go @@ -37,6 +37,7 @@ import ( "sort" "strconv" "strings" + "text/template" "time" "github.com/google/go-github/github" @@ -47,6 +48,12 @@ import ( var maxPayloadSize = 1024 * 1024 * 55 // 55 MB type submitServer struct { + // Template for building github and gitlab issues + issueTemplate *template.Template + + // Template for building emails + emailTemplate *template.Template + // github client for reporting bugs. may be nil, in which case, // reporting is disabled. ghClient *github.Client @@ -78,6 +85,16 @@ type jsonLogEntry struct { Lines string `json:"lines"` } +// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template` and +// `email_body_template`. +// +// !!! Keep in step with the documentation in `templates/README.md` !!! +type issueBodyTemplatePayload struct { + payload + // Complete link to the listing URL that contains all uploaded logs + ListingURL string +} + // Stores additional information created during processing of a payload type genericWebhookPayload struct { payload @@ -87,7 +104,10 @@ type genericWebhookPayload struct { ListingURL string `json:"listing_url"` } -// Stores information about a request made to this server +// `payload` stores information about a request made to this server. +// +// !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step +// with the documentation in `templates/README.md` !!! type payload struct { // A unique ID for this payload, generated within this server ID string `json:"id"` @@ -505,7 +525,7 @@ func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, lis return nil, err } - if err := s.sendEmail(p, reportDir); err != nil { + if err := s.sendEmail(p, reportDir, listingURL); err != nil { return nil, err } @@ -580,9 +600,12 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing } owner, repo := splits[0], splits[1] - issueReq := buildGithubIssueRequest(p, listingURL) + issueReq, err := buildGithubIssueRequest(p, listingURL, s.issueTemplate) + if err != nil { + return err + } - issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq) + issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, issueReq) if err != nil { return err } @@ -602,7 +625,10 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub glProj := s.cfg.GitlabProjectMappings[p.AppName] glLabels := s.cfg.GitlabProjectLabels[p.AppName] - issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential) + issueReq, err := buildGitlabIssueRequest(p, listingURL, s.issueTemplate, glLabels, s.cfg.GitlabIssueConfidential) + if err != nil { + return err + } issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq) @@ -649,80 +675,72 @@ func buildReportTitle(p payload) string { return trimmedUserText } -func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer { +func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title string, body []byte, err error) { var bodyBuf bytes.Buffer - fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText) - var dataKeys []string - for k := range p.Data { - dataKeys = append(dataKeys, k) - } - sort.Strings(dataKeys) - for _, k := range dataKeys { - v := p.Data[k] - fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline) - } - - return &bodyBuf -} -func buildGenericIssueRequest(p payload, listingURL string) (title, body string) { - bodyBuf := buildReportBody(p, " \n", "`") - - // Add log links to the body - fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL) - fmt.Fprintf(bodyBuf, " ([archive](%s))", listingURL+"?format=tar.gz") + issuePayload := issueBodyTemplatePayload{ + payload: p, + ListingURL: listingURL, + } - for _, file := range p.Files { - fmt.Fprintf( - bodyBuf, - " / [%s](%s)", - file, - listingURL+"/"+file, - ) + if err = bodyTemplate.Execute(&bodyBuf, issuePayload); err != nil { + return } title = buildReportTitle(p) - - body = bodyBuf.String() + body = bodyBuf.Bytes() return } -func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest { - title, body := buildGenericIssueRequest(p, listingURL) +func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (*github.IssueRequest, error) { + title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) + if err != nil { + return nil, err + } labels := p.Labels // go-github doesn't like nils if labels == nil { labels = []string{} } - return github.IssueRequest{ + bodyStr := string(body) + return &github.IssueRequest{ Title: &title, - Body: &body, + Body: &bodyStr, Labels: &labels, - } + }, nil } -func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions { - title, body := buildGenericIssueRequest(p, listingURL) +func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *template.Template, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) { + title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate) + if err != nil { + return nil, err + } if p.Labels != nil { labels = append(labels, p.Labels...) } + bodyStr := string(body) return &gitlab.CreateIssueOptions{ Title: &title, - Description: &body, + Description: &bodyStr, Confidential: &confidential, Labels: labels, - } + }, nil } -func (s *submitServer) sendEmail(p payload, reportDir string) error { +func (s *submitServer) sendEmail(p payload, reportDir string, listingURL string) error { if len(s.cfg.EmailAddresses) == 0 { return nil } + title, body, err := buildGenericIssueRequest(p, listingURL, s.emailTemplate) + if err != nil { + return err + } + e := email.NewEmail() e.From = "Rageshake " @@ -731,10 +749,8 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error { } e.To = s.cfg.EmailAddresses - - e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p)) - - e.Text = buildReportBody(p, "\n", "\"").Bytes() + e.Subject = fmt.Sprintf("[%s] %s", p.AppName, title) + e.Text = body allFiles := append(p.Files, p.Logs...) for _, file := range allFiles { @@ -746,7 +762,7 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error { if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" { auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer) } - err := e.Send(s.cfg.SMTPServer, auth) + err = e.Send(s.cfg.SMTPServer, auth) if err != nil { return err } diff --git a/submit_test.go b/submit_test.go index aa54a33..12ebefe 100644 --- a/submit_test.go +++ b/submit_test.go @@ -28,6 +28,7 @@ import ( "strconv" "strings" "testing" + "text/template" ) // testParsePayload builds a /submit request with the given body, and calls @@ -56,11 +57,10 @@ func testParsePayload(t *testing.T, body, contentType string, tempDir string) (* return p, rr.Result() } - func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool, body string) int { - // Submit a request without files to the server and return statusCode - // Could be extended with more complicated config; aimed here just to - // test options for allowedAppNameMap + // Submit a request without files to the server and return statusCode + // Could be extended with more complicated config; aimed here just to + // test options for allowedAppNameMap req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body)) if err != nil { @@ -70,7 +70,7 @@ func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool w := httptest.NewRecorder() var cfg config - s := &submitServer{nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} + s := &submitServer{nil, nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg} s.ServeHTTP(w, req) rsp := w.Result() @@ -406,6 +406,63 @@ func mkTempDir(t *testing.T) string { * buildGithubIssueRequest tests */ +// General test of Github issue formatting. +func TestBuildGithubIssue(t *testing.T) { + body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="text" + + +test words. +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="app" + +riot-web +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="User-Agent" + +xxx +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="user_id" + +id +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="device_id" + +id +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="version" + +1 +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="file"; filename="passwd.txt" + +file +------WebKitFormBoundarySsdgl8Nq9voFyhdO-- +` + p, _ := testParsePayload(t, body, + "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", + "", + ) + + if p == nil { + t.Fatal("parseRequest returned nil") + } + + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } + + if *issueReq.Title != "test words." { + t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") + } + expectedBody := "User message:\n\ntest words.\n\nUser-Agent: `xxx`\nVersion: `1`\ndevice_id: `id`\nuser_id: `id`\n\n[Logs](http://test/listing/foo) ([archive](http://test/listing/foo?format=tar.gz)) / [passwd.txt](http://test/listing/foo/passwd.txt)\n" + if *issueReq.Body != expectedBody { + t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) + } +} + func TestBuildGithubIssueLeadingNewline(t *testing.T) { body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO Content-Disposition: form-data; name="text" @@ -427,12 +484,16 @@ riot-web t.Fatal("parseRequest returned nil") } - issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo") + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } if *issueReq.Title != "test words." { t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.") } - expectedBody := "User message:\n\n\ntest words.\n" + expectedBody := "User message:\n\ntest words.\n" if !strings.HasPrefix(*issueReq.Body, expectedBody) { t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody) } @@ -453,7 +514,11 @@ Content-Disposition: form-data; name="text" t.Fatal("parseRequest returned nil") } - issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo") + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) + issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } if *issueReq.Title != "Untitled report" { t.Errorf("Title: got %s, want %s", *issueReq.Title, "Untitled report") @@ -464,7 +529,7 @@ Content-Disposition: form-data; name="text" } } -func TestTestSortDataKeys(t *testing.T) { +func TestSortDataKeys(t *testing.T) { expect := ` Number of logs: 0 Application: @@ -506,9 +571,13 @@ user_id: id } } + parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate)) for k, v := range sample { p := payload{Data: v.data} - res := buildGithubIssueRequest(p, "") + res, err := buildGithubIssueRequest(p, "", parsedIssueTemplate) + if err != nil { + t.Fatalf("Error building issue request: %s", err) + } got := *res.Body if k == 0 { expect = got diff --git a/templates/README.md b/templates/README.md new file mode 100644 index 0000000..9382e2c --- /dev/null +++ b/templates/README.md @@ -0,0 +1,26 @@ +This directory contains the default templates that are used by the rageshake server. + +The templates can be overridden via settings in the config file. + +The templates are as follows: + +* `issue_body.tmpl`: Used when filing an issue at Github or Gitlab, and gives the issue description. Override via + the `issue_body_template_file` setting in the configuration file. +* `email_body.tmpl`: Used when sending an email. Override via the `email_body_template_file` configuration setting. + +See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language. + +The following properties are defined on the input (accessible via `.` or `$`): + +| Name | Type | Description | +|--------------|---------------------|---------------------------------------------------------------------------------------------------| +| `ID` | `string` | The unique ID for this rageshake. | +| `UserText` | `string` | A multi-line string containing the user description of the fault (from `text` in the submission). | +| `AppName` | `string` | A short slug to identify the app making the report (from `app` in the submission). | +| `Labels` | `[]string` | A list of labels requested by the application. | +| `Data` | `map[string]string` | A map of other key/value pairs included in the submission. | +| `Logs` | `[]string` | A list of log file names. | +| `LogErrors` | `[]string` | Set if there are log parsing errors. | +| `Files` | `[]string` | A list of other files (not logs) uploaded as part of the rageshake. | +| `FileErrors` | `[]string` | Set if there are file parsing errors. | +| `ListingURL` | `string` | Complete link to the listing URL that contains all uploaded logs. | diff --git a/templates/email_body.tmpl b/templates/email_body.tmpl new file mode 100644 index 0000000..13cbab2 --- /dev/null +++ b/templates/email_body.tmpl @@ -0,0 +1,9 @@ +User message: +{{ .UserText }} + +{{ range $key, $val := .Data -}} +{{ $key }}: "{{ $val }}" +{{ end }} +[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) +{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) +{{- end }} diff --git a/templates/issue_body.tmpl b/templates/issue_body.tmpl new file mode 100644 index 0000000..8de4d52 --- /dev/null +++ b/templates/issue_body.tmpl @@ -0,0 +1,9 @@ +User message: +{{ .UserText }} + +{{ range $key, $val := .Data -}} +{{ $key }}: `{{ $val }}` +{{ end }} +[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz)) +{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }}) +{{- end }}