di: finish di timeline

coverage/coverage.go

@@ -0,0 +1,68 @@
+package coverage
+
+import (
+    "database/sql"
+    "encoding/json"
+    "io/ioutil"
+    "net/http"
+    "os"
+    "strconv"
+    "strings"
+    "time"
+)
+
+// ProcessCoverage gets the coverage of {owner}/{repo} after each commit in the past year through codecov's API, saves into mysql
+func ProcessCoverage(owner, repo string) error {
+    client := http.Client{}
+    req, err := http.NewRequest("GET", "https://codecov.io/api/gh/"+owner+"/"+repo+"/branch/master/graphs/commits.json?method=min&agg=day&time=365d&inc=totals&order=asc", strings.NewReader(""))
+    if err != nil {
+        return err
+    }
+
+    req.Header.Set("Authorization", "token a3a4a9847e4b4e30b1ddc4dd725bf110")
+
+    resp, err := client.Do(req)
+    if err != nil {
+        return err
+    }
+
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        return err
+    }
+
+    var message interface{}
+
+    json.Unmarshal(body, &message)
+
+    db, err := sql.Open("mysql", os.Getenv("GITHUB_DSN"))
+    if err != nil {
+        return err
+    }
+    tx, err := db.Begin()
+    if err != nil {
+        return err
+    }
+
+    commits := message.(map[string]interface{})["commits"]
+    for _, commit := range commits.([]interface{}) {
+        timestamp := commit.(map[string]interface{})["timestamp"]
+        totals := commit.(map[string]interface{})["totals"]
+        coverage, err := strconv.ParseFloat(totals.(map[string]interface{})["c"].(string), 64)
+        if err != nil {
+            return err
+        }
+        t, err := time.Parse("2006-01-02 15:04:05", timestamp.(string))
+        if err != nil {
+            return err
+        }
+        _, err = tx.Exec("INSERT INTO COVERAGE_TIMELINE(REPO, TIME, COVERAGE) VALUES(?, ?, ?)", "pd", t, coverage)
+        if err != nil {
+            return err
+        }
+    }
+
+    tx.Commit()
+
+    return nil
+}

coverage/coverage_test.go

@@ -0,0 +1,10 @@
+package coverage
+
+import "testing"
+
+func TestProcessCoverage(t *testing.T) {
+    err := ProcessCoverage("pingcap", "tidb")
+    if err != nil {
+        t.Fatal(err)
+    }
+}

di/computing.go

@@ -14,11 +14,8 @@
 package di
 
 import (
-    "context"
     "database/sql"
-    "errors"
     "log"
-    "strings"
     "time"
 )
 
@@ -45,12 +42,14 @@ type Issue struct {
     Label        map[string][]string
 }
 
+// Interval DI struct
 type IntervalDI struct {
     StartTime time.Time
     EndTime   time.Time
     Value     float64
 }
 
+// Instant DI struct
 type InstantDI struct {
     Time  time.Time
     Value float64
@@ -87,47 +86,6 @@ func calculateDI(issues []Issue) float64 {
     return di
 }
 
-// getLabels returns all labels of an issue, saved in map.
-func getLabels(db *sql.DB, issue Issue) (map[string][]string, error) {
-
-    if db == nil {
-        return nil, errors.New("db is nil")
-    }
-
-    labels := make(map[string][]string)
-
-    ctx, cancel := context.WithTimeout(context.Background(), mysqlQueryTimeout)
-    defer cancel()
-    rows, err := db.QueryContext(ctx, `SELECT NAME 
-                                        	  FROM LABEL_ISSUE_RELATIONSHIP, LABEL 
-                                        	  WHERE LABEL_ISSUE_RELATIONSHIP.ISSUE_ID = ? 
-                                       		  AND LABEL_ISSUE_RELATIONSHIP.LABEL_ID = LABEL.ID`, issue.ID)
-
-    if err != nil {
-        return nil, err
-    }
-
-    for rows.Next() {
-        var label string
-        err := rows.Scan(&label)
-
-        if err != nil {
-            return nil, err
-        }
-        parts := strings.Split(label, "/")
-        switch len(parts) {
-        case 1:
-            labels[parts[0]] = append(labels[parts[0]], "")
-        case 2:
-            labels[parts[0]] = append(labels[parts[0]], parts[1])
-        default:
-            log.Printf("Issue %v has unsupported label %s", issue.Number, label)
-        }
-    }
-
-    return labels, nil
-}
-
 // parseIssues returns issues parsed from sql.Rows
 func parseIssues(rows *sql.Rows) ([]Issue, error) {
     issues := make([]Issue, 0)
@@ -144,153 +102,4 @@ func parseIssues(rows *sql.Rows) ([]Issue, error) {
     }
 
     return issues, nil
-}
-
-// getCreatedDIBetweenTime returns repo's DI of issues created between startTime and endTime
-// if repo is empty string, all repo's DI will be involved
-func getCreatedDIBetweenTime(db *sql.DB, startTime, endTime time.Time) (float64, error) {
-    if db == nil {
-        return 0, errors.New("db is nil")
-    }
-
-    if startTime.After(endTime) {
-        return 0, errors.New("startTime > endTime")
-    }
-
-    ctx, cancel := context.WithTimeout(context.Background(), mysqlQueryTimeout)
-    defer cancel()
-
-    rows, err := db.QueryContext(ctx, "SELECT ID, NUMBER FROM ISSUE WHERE CREATED_AT BETWEEN ? AND ?", startTime, endTime)
-
-    if err != nil {
-        return 0, err
-    }
-
-    issues, err := parseIssues(rows)
-    if err != nil {
-        return 0, err
-    }
-
-    for i, _ := range issues {
-        issues[i].Label, err = getLabels(db, issues[i])
-        if err != nil {
-            return 0, err
-        }
-    }
-
-    di := calculateDI(issues)
-
-    return di, nil
-}
-
-// getClosedDIBetweenTime returns total DI of issues closed between startTime and endTime
-func getClosedDIBetweenTime(db *sql.DB, startTime, endTime time.Time) (float64, error) {
-    if db == nil {
-        return 0, errors.New("db is nil")
-    }
-
-    if startTime.After(endTime) {
-        return 0, errors.New("startTime > endTime")
-    }
-
-    ctx, cancel := context.WithTimeout(context.Background(), mysqlQueryTimeout)
-    defer cancel()
-    rows, err := db.QueryContext(ctx, "SELECT ID, NUMBER FROM ISSUE WHERE CLOSED_AT BETWEEN ? AND ?", startTime, endTime)
-    if err != nil {
-        return 0, err
-    }
-
-    issues, err := parseIssues(rows)
-    if err != nil {
-        return 0, err
-    }
-
-    for i, _ := range issues {
-        issues[i].Label, err = getLabels(db, issues[i])
-        if err != nil {
-            return 0, err
-        }
-    }
-
-    di := calculateDI(issues)
-
-    return di, nil
-}
-
-// getDI returns DI at a specified time
-func getDI(db *sql.DB, time time.Time) (float64, error) {
-    if db == nil {
-        return 0, errors.New("db is nil")
-    }
-
-    ctx, cancel := context.WithTimeout(context.Background(), mysqlQueryTimeout)
-    defer cancel()
-    rows, err := db.QueryContext(ctx, "SELECT ID, NUMBER FROM ISSUE WHERE CREATED_AT < ? AND (CLOSED = 0 OR CLOSED_AT > ?)", time, time)
-    if err != nil {
-        return 0, err
-    }
-
-    issues, err := parseIssues(rows)
-    if err != nil {
-        return 0, err
-    }
-
-    for i, _ := range issues {
-        issues[i].Label, err = getLabels(db, issues[i])
-        if err != nil {
-            return 0, err
-        }
-    }
-
-    di := calculateDI(issues)
-
-    return di, nil
-}
-
-// GetCreatedDIsFrom gets issue information from db, returns CreatedDI of each interval of fixed length from a specified time
-func getCreatedDIsFrom(db *sql.DB, startTime time.Time, frequency time.Duration) ([]IntervalDI, error) {
-    dis := make([]IntervalDI, 0)
-
-    for startTime.Before(time.Now()) {
-        endTime := startTime.Add(frequency)
-
-        di, err := getCreatedDIBetweenTime(db, startTime, endTime)
-        if err != nil {
-            return nil, err
-        }
-
-        dis = append(dis, IntervalDI{
-            StartTime: startTime,
-            EndTime:   endTime,
-            Value:     di,
-        })
-
-        startTime = endTime
-    }
-
-    return dis, nil
-}
-
-// getCreatedDIsFrom gets issue information from db, returns CreatedDI of each interval of fixed length from a specified time
-func getClosedDIsFrom(db *sql.DB, startTime time.Time, frequency time.Duration) ([]IntervalDI, error) {
-    dis := make([]IntervalDI, 0)
-
-    for startTime.Before(time.Now()) {
-        endTime := startTime.Add(frequency)
-
-        di, err := getClosedDIBetweenTime(db, startTime, endTime)
-        if err != nil {
-            return nil, err
-        }
-
-        dis = append(dis, IntervalDI{
-            StartTime: startTime,
-            EndTime:   endTime,
-            Value:     di,
-        })
-
-        startTime = endTime
-    }
-
-    return dis, nil
-}
+}

di/computing_test.go

@@ -16,7 +16,6 @@ package di
 import (
     "database/sql"
     "log"
-    "math"
     "os"
     "testing"
     "time"
@@ -26,18 +25,6 @@ import (
 
 var issueDB *sql.DB
 
-func must(t *testing.T, value interface{}, expected interface{}, name string) {
-    if expected != value {
-        t.Fatalf("%v = %v, expected %v", name, value, expected)
-    }
-}
-
-func mustNot(t *testing.T, value interface{}, unexpected interface{}, name string) {
-    if unexpected == value {
-        t.Fatalf("%v must not be %v", name, unexpected)
-    }
-}
-
 func init() {
     dsn := os.Getenv("GITHUB_DSN")
     var err error
@@ -72,52 +59,3 @@ func TestCalculateDi(t *testing.T) {
     must(t, di, 0.0, "di")
 
 }
-
-func TestGetCreatedDiBetweenTime(t *testing.T) {
-    startTime := time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC)
-    endTime := time.Date(2015, 11, 1, 0, 0, 0, 0, time.UTC)
-    di, err := getCreatedDIBetweenTime(issueDB, startTime, endTime)
-    must(t, err, nil, "err")
-    must(t, di, 0.1, "di")
-
-    startTime = time.Date(2020, 10, 27, 0, 0, 0, 0, time.UTC)
-    endTime = time.Date(2020, 10, 26, 0, 0, 0, 0, time.UTC)
-    _, err = getCreatedDIBetweenTime(issueDB, startTime, endTime)
-    mustNot(t, err, nil, "err")
-}
-
-func TestGetClosedDiBetweenTime(t *testing.T) {
-    startTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
-    endTime := time.Date(2020, 11, 1, 0, 0, 0, 0, time.UTC)
-    di, err := getClosedDIBetweenTime(issueDB, startTime, endTime)
-    must(t, err, nil, "err")
-    must(t, di, 139.1, "di")
-
-    startTime = time.Date(2020, 10, 27, 0, 0, 0, 0, time.UTC)
-    endTime = time.Date(2020, 10, 26, 0, 0, 0, 0, time.UTC)
-    di, err = getClosedDIBetweenTime(issueDB, startTime, endTime)
-    mustNot(t, err, nil, "err")
-}
-
-func TestGetCreatedDIsFrom(t *testing.T) {
-    ti := time.Date(2020, 9, 21, 0, 0, 0, 0, time.UTC)
-    dis, err := getCreatedDIsFrom(issueDB, ti, 7*24*time.Hour)
-    must(t, err, nil, "err")
-    must(t, len(dis), 7, `len(dis)`)
-}
-
-func TestGetClosedDIsFrom(t *testing.T) {
-    ti := time.Date(2020, 9, 21, 0, 0, 0, 0, time.UTC)
-    dis, err := getClosedDIsFrom(issueDB, ti, 7*24*time.Hour)
-    must(t, err, nil, "err")
-    must(t, len(dis), 7, `len(dis)`)
-}
-
-func TestGetDI(t *testing.T) {
-    ti := time.Date(2020, 9, 21, 0, 0, 0, 0, time.UTC)
-    di, err := getDI(issueDB, ti)
-    must(t, err, nil, "err")
-    if math.Abs(di-1229.3) > 1e-6 {
-        t.Fatalf("GetDI %v returns %f", ti, di)
-    }
-}