chore: include extra columns for errors reporting table (#4891)

* chore: include extra columns for errors reporting table * chore: update insertion logic to include new columns - add db migration script to include new columns in error_detail_reports table - refactor reporting types * chore: fix alter query and send sampleEvent as string in Report fn * chore: refactor error reporting types * fix: update count logic * chore: lint errors --------- Co-authored-by: Sai Sankeerth <[email protected]> Co-authored-by: Dilip Kola <[email protected]>
rudderlabs · Jul 22, 2024 · 6ca22ca · 6ca22ca
1 parent 009a071
commit 6ca22ca
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 61 deletions.
diff --git a/enterprise/reporting/error_reporting.go b/enterprise/reporting/error_reporting.go
@@ -50,6 +50,9 @@ var ErrorDetailReportsColumns = []string{
 	"event_type",
 	"error_code",
 	"error_message",
+	"sample_response",
+	"sample_event",
+	"event_name",
 }
 
 type ErrorDetailReporter struct {
@@ -218,6 +221,9 @@ func (edr *ErrorDetailReporter) Report(ctx context.Context, metrics []*types.PUR
 			metric.StatusDetail.EventType,
 			errDets.ErrorCode,
 			errDets.ErrorMessage,
+			metric.StatusDetail.SampleResponse,
+			string(metric.StatusDetail.SampleEvent),
+			metric.StatusDetail.EventName,
 		)
 		if err != nil {
 			edr.log.Errorf("Failed during statement execution(each metric): %v", err)
@@ -414,6 +420,9 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
 		"error_code",
 		"error_message",
 		"dest_type",
+		"sample_response",
+		"sample_event",
+		"event_name",
 	}, ", ")
 	var rows *sql.Rows
 	queryStart = time.Now()
@@ -443,6 +452,9 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
 			"error_code",
 			"error_message",
 			"dest_type",
+			"sample_response",
+			"sample_event",
+			"event_name",
 		*/
 		dbEdMetric := &types.EDReportsDB{
 			EDErrorDetails:    types.EDErrorDetails{},
@@ -459,11 +471,14 @@ func (edr *ErrorDetailReporter) getReports(ctx context.Context, currentMs int64,
 			&dbEdMetric.PU,
 			&dbEdMetric.ReportMetadata.ReportedAt,
 			&dbEdMetric.Count,
-			&dbEdMetric.EDErrorDetails.StatusCode,
-			&dbEdMetric.EDErrorDetails.EventType,
-			&dbEdMetric.EDErrorDetails.ErrorCode,
-			&dbEdMetric.EDErrorDetails.ErrorMessage,
+			&dbEdMetric.EDErrorDetails.EDErrorDetailsKey.StatusCode,
+			&dbEdMetric.EDErrorDetails.EDErrorDetailsKey.EventType,
+			&dbEdMetric.EDErrorDetails.EDErrorDetailsKey.ErrorCode,
+			&dbEdMetric.EDErrorDetails.EDErrorDetailsKey.ErrorMessage,
 			&dbEdMetric.EDConnectionDetails.DestType,
+			&dbEdMetric.EDErrorDetails.SampleResponse,
+			&dbEdMetric.EDErrorDetails.SampleEvent,
+			&dbEdMetric.EDErrorDetails.EDErrorDetailsKey.EventName,
 		)
 		if err != nil {
 			edr.log.Errorf("Failed while scanning rows(reported_at=%v): %v", queryMin.Int64, err)
@@ -519,16 +534,20 @@ func (edr *ErrorDetailReporter) aggregate(reports []*types.EDReportsDB) []*types
 			},
 		}
 		messageMap := make(map[string]int)
-		reportsCountMap := make(map[types.EDErrorDetails]int64)
+		reportsCountMap := make(map[types.EDErrorDetailsKey]*types.EDReportMapValue)
 		for index, rep := range reports {
 			messageMap[rep.EDErrorDetails.ErrorMessage] = index
-			errDet := types.EDErrorDetails{
-				StatusCode:   rep.StatusCode,
-				ErrorCode:    rep.ErrorCode,
-				ErrorMessage: rep.ErrorMessage,
-				EventType:    rep.EventType,
+			errDet := rep.EDErrorDetails.EDErrorDetailsKey
+			reportMapValue, ok := reportsCountMap[errDet]
+			if !ok {
+				reportsCountMap[errDet] = &types.EDReportMapValue{
+					SampleResponse: rep.SampleResponse,
+					SampleEvent:    rep.SampleEvent,
+					Count:          rep.Count,
+				}
+				continue
 			}
-			reportsCountMap[errDet] += rep.Count
+			reportMapValue.Count += rep.Count
 		}
 
 		reportGrpKeys := lo.Keys(reportsCountMap)
@@ -542,13 +561,12 @@ func (edr *ErrorDetailReporter) aggregate(reports []*types.EDReportsDB) []*types
 		})
 		errs := make([]types.EDErrorDetails, len(reportGrpKeys))
 		for i, repKey := range reportGrpKeys {
-			repCount := reportsCountMap[repKey]
+			repValue := reportsCountMap[repKey]
 			errs[i] = types.EDErrorDetails{
-				StatusCode:   repKey.StatusCode,
-				ErrorCode:    repKey.ErrorCode,
-				ErrorMessage: repKey.ErrorMessage,
-				EventType:    repKey.EventType,
-				Count:        repCount,
+				EDErrorDetailsKey: repKey,
+				SampleResponse:    repValue.SampleResponse,
+				SampleEvent:       repValue.SampleEvent,
+				ErrorCount:        repValue.Count,
 			}
 		}
 		edrSchema.Errors = errs

diff --git a/enterprise/reporting/reporting_test.go b/enterprise/reporting/reporting_test.go
@@ -476,10 +476,12 @@ func TestAggregationLogic(t *testing.T) {
 				DestType:                "DES_1",
 			},
 			EDErrorDetails: types.EDErrorDetails{
-				StatusCode:   200,
-				ErrorCode:    "",
-				ErrorMessage: "",
-				EventType:    "identify",
+				EDErrorDetailsKey: types.EDErrorDetailsKey{
+					StatusCode:   200,
+					ErrorCode:    "",
+					ErrorMessage: "",
+					EventType:    "identify",
+				},
 			},
 			ReportMetadata: types.ReportMetadata{
 				ReportedAt: 124335445,
@@ -501,10 +503,12 @@ func TestAggregationLogic(t *testing.T) {
 				DestType:                "DES_1",
 			},
 			EDErrorDetails: types.EDErrorDetails{
-				StatusCode:   400,
-				ErrorCode:    "",
-				ErrorMessage: "bad data sent for transformation",
-				EventType:    "identify",
+				EDErrorDetailsKey: types.EDErrorDetailsKey{
+					StatusCode:   400,
+					ErrorCode:    "",
+					ErrorMessage: "bad data sent for transformation",
+					EventType:    "identify",
+				},
 			},
 			ReportMetadata: types.ReportMetadata{
 				ReportedAt: 124335445,
@@ -526,10 +530,12 @@ func TestAggregationLogic(t *testing.T) {
 				DestType:                "DES_1",
 			},
 			EDErrorDetails: types.EDErrorDetails{
-				StatusCode:   400,
-				ErrorCode:    "",
-				ErrorMessage: "bad data sent for transformation",
-				EventType:    "identify",
+				EDErrorDetailsKey: types.EDErrorDetailsKey{
+					StatusCode:   400,
+					ErrorCode:    "",
+					ErrorMessage: "bad data sent for transformation",
+					EventType:    "identify",
+				},
 			},
 			ReportMetadata: types.ReportMetadata{
 				ReportedAt: 124335445,
@@ -551,10 +557,12 @@ func TestAggregationLogic(t *testing.T) {
 				DestType:                "DES_1",
 			},
 			EDErrorDetails: types.EDErrorDetails{
-				StatusCode:   400,
-				ErrorCode:    "",
-				ErrorMessage: "user_id information missing",
-				EventType:    "identify",
+				EDErrorDetailsKey: types.EDErrorDetailsKey{
+					StatusCode:   400,
+					ErrorCode:    "",
+					ErrorMessage: "user_id information missing",
+					EventType:    "identify",
+				},
 			},
 			ReportMetadata: types.ReportMetadata{
 				ReportedAt: 124335446,
@@ -577,10 +585,12 @@ func TestAggregationLogic(t *testing.T) {
 				DestType:                "DES_1",
 			},
 			EDErrorDetails: types.EDErrorDetails{
-				StatusCode:   500,
-				ErrorCode:    "",
-				ErrorMessage: "Cannot read type property of undefined", // some error during batching
-				EventType:    "identify",
+				EDErrorDetailsKey: types.EDErrorDetailsKey{
+					StatusCode:   500,
+					ErrorCode:    "",
+					ErrorMessage: "Cannot read type property of undefined", // some error during batching
+					EventType:    "identify",
+				},
 			},
 			ReportMetadata: types.ReportMetadata{
 				ReportedAt: 124335446,
@@ -612,18 +622,22 @@ func TestAggregationLogic(t *testing.T) {
 			},
 			Errors: []types.EDErrorDetails{
 				{
-					StatusCode:   dbErrs[0].StatusCode,
-					ErrorCode:    dbErrs[0].ErrorCode,
-					ErrorMessage: dbErrs[0].ErrorMessage,
-					EventType:    dbErrs[0].EventType,
-					Count:        10,
+					EDErrorDetailsKey: types.EDErrorDetailsKey{
+						StatusCode:   dbErrs[0].StatusCode,
+						ErrorCode:    dbErrs[0].ErrorCode,
+						ErrorMessage: dbErrs[0].ErrorMessage,
+						EventType:    dbErrs[0].EventType,
+					},
+					ErrorCount: 10,
 				},
 				{
-					StatusCode:   dbErrs[1].StatusCode,
-					ErrorCode:    dbErrs[1].ErrorCode,
-					ErrorMessage: dbErrs[1].ErrorMessage,
-					EventType:    dbErrs[1].EventType,
-					Count:        20,
+					EDErrorDetailsKey: types.EDErrorDetailsKey{
+						StatusCode:   dbErrs[1].StatusCode,
+						ErrorCode:    dbErrs[1].ErrorCode,
+						ErrorMessage: dbErrs[1].ErrorMessage,
+						EventType:    dbErrs[1].EventType,
+					},
+					ErrorCount: 20,
 				},
 			},
 		},
@@ -646,11 +660,13 @@ func TestAggregationLogic(t *testing.T) {
 			},
 			Errors: []types.EDErrorDetails{
 				{
-					StatusCode:   dbErrs[3].StatusCode,
-					ErrorCode:    dbErrs[3].ErrorCode,
-					ErrorMessage: dbErrs[3].ErrorMessage,
-					EventType:    dbErrs[3].EventType,
-					Count:        20,
+					EDErrorDetailsKey: types.EDErrorDetailsKey{
+						StatusCode:   dbErrs[3].StatusCode,
+						ErrorCode:    dbErrs[3].ErrorCode,
+						ErrorMessage: dbErrs[3].ErrorMessage,
+						EventType:    dbErrs[3].EventType,
+					},
+					ErrorCount: 20,
 				},
 			},
 		},
@@ -673,11 +689,13 @@ func TestAggregationLogic(t *testing.T) {
 			},
 			Errors: []types.EDErrorDetails{
 				{
-					StatusCode:   dbErrs[4].StatusCode,
-					ErrorCode:    dbErrs[4].ErrorCode,
-					ErrorMessage: dbErrs[4].ErrorMessage,
-					EventType:    dbErrs[4].EventType,
-					Count:        15,
+					EDErrorDetailsKey: types.EDErrorDetailsKey{
+						StatusCode:   dbErrs[4].StatusCode,
+						ErrorCode:    dbErrs[4].ErrorCode,
+						ErrorMessage: dbErrs[4].ErrorMessage,
+						EventType:    dbErrs[4].EventType,
+					},
+					ErrorCount: 15,
 				},
 			},
 		},

diff --git a/sql/migrations/error_detail_reports/000002_alter_error_detail_reports.up.sql b/sql/migrations/error_detail_reports/000002_alter_error_detail_reports.up.sql
@@ -0,0 +1,4 @@
+ALTER TABLE error_detail_reports
+ADD COLUMN IF NOT EXISTS sample_event JSONB DEFAULT '{}'::JSONB,
+ADD COLUMN IF NOT EXISTS sample_response TEXT DEFAULT '',
+ADD COLUMN IF NOT EXISTS event_name TEXT DEFAULT '';
diff --git a/utils/types/reporting_types.go b/utils/types/reporting_types.go
@@ -108,13 +108,19 @@ type EDInstanceDetails struct {
 	InstanceID  string `json:"-"`
 }
 
-type EDErrorDetails struct {
+type EDErrorDetailsKey struct {
 	StatusCode   int    `json:"statusCode"`
 	ErrorCode    string `json:"errorCode"`
 	ErrorMessage string `json:"errorMessage"`
-	// TODO: need to check with team if this makes sense ?
-	EventType string `json:"-"`
-	Count     int64  `json:"count"`
+	EventType    string `json:"eventType"`
+	EventName    string `json:"eventName"`
+}
+
+type EDErrorDetails struct {
+	EDErrorDetailsKey
+	SampleResponse string          `json:"sampleResponse"`
+	SampleEvent    json.RawMessage `json:"sampleEvent"`
+	ErrorCount     int64           `json:"count"`
 }
 
 type EDReportsDB struct {
@@ -127,6 +133,12 @@ type EDReportsDB struct {
 	Count int64  `json:"-"`
 }
 
+type EDReportMapValue struct {
+	Count          int64
+	SampleResponse string
+	SampleEvent    json.RawMessage
+}
+
 // EDMetric The structure in which the error detail data is being sent to reporting service
 type EDMetric struct {
 	EDInstanceDetails