Merge pull request #22 from vshn/rewrite

Complete Wrestic rewrite

.vscode/launch.json

@@ -18,11 +18,10 @@
             "RESTIC_PASSWORD": "asdf",
             "AWS_ACCESS_KEY_ID": "8U0UDNYPNUDTUS1LIAF3",
             "AWS_SECRET_ACCESS_KEY": "ip3cdrkXcHmH4S7if7erKPNoxDn27V0vrg6CHHem",
-            "PROM_URL":"http://localhost:9091/",
-            "HOSTNAME":"test",
-            "BACKUP_DIR": "/Users/simonbeck/Desktop/data",
-            "STATS_URL": "http://localhost:8091",
-            "NAMESPACE": "myproject",
+            "PROM_URL":"http://localhost:9000/",
+            "HOSTNAME":"myproject",
+            "BACKUP_DIR": "/Volumes/VMs/baas/data",
+            "STATS_URL": "http://localhost:9000",
         }
       },
       {
@@ -81,24 +80,24 @@
         "program": "${workspaceFolder}/cmd/wrestic/main.go",
         "port": 2345,
         "host": "127.0.0.1",
-        "args":[
-            "-restore",
-            "-restoreType",
-            "s3",
-            "-restoreSnap",
-            "7a24ca037aa01b9ceaf3d4be189dddb2ecfedf603217970a8f2027609797f4b1",
-        ],
+        // "args":[
+        //     "-restore",
+        //     "-restoreType",
+        //     "s3",
+        //     "-restoreSnap",
+        //     "7a24ca037aa01b9ceaf3d4be189dddb2ecfedf603217970a8f2027609797f4b1",
+        // ],
         "env": {
-            "RESTIC_REPOSITORY": "s3:http://localhost:9000/baas",
-            "RESTIC_PASSWORD": "password",
-            "AWS_ACCESS_KEY_ID": "8U0UDNYPNUDTUS1LIAF3",
-            "AWS_SECRET_ACCESS_KEY": "ip3cdrkXcHmH4S7if7erKPNoxDn27V0vrg6CHHem",
+            "RESTIC_REPOSITORY": "s3:http://localhost:9000/baas1",
+            "RESTIC_PASSWORD": "asdf",
+            "AWS_ACCESS_KEY_ID": "minioadmin",
+            "AWS_SECRET_ACCESS_KEY": "minioadmin",
             "PROM_URL":"http://localhost:9091/",
             "HOSTNAME":"test",
             "BACKUP_DIR": "/Users/simonbeck/Desktop/data",
             "RESTORE_S3ENDPOINT":"http://localhost:9000/restore",
-            "RESTORE_ACCESSKEYID":"8U0UDNYPNUDTUS1LIAF3",
-            "RESTORE_SECRETACCESSKEY": "ip3cdrkXcHmH4S7if7erKPNoxDn27V0vrg6CHHem",
+            "RESTORE_ACCESSKEYID":"minioadmin",
+            "RESTORE_SECRETACCESSKEY": "minioadmin",
             "STATS_URL": "http://localhost:8091",
         }
       },
@@ -119,8 +118,8 @@
         "env": {
             "RESTIC_REPOSITORY": "s3:http://localhost:9000/baas1",
             "RESTIC_PASSWORD": "asdf",
-            "AWS_ACCESS_KEY_ID": "8U0UDNYPNUDTUS1LIAF3",
-            "AWS_SECRET_ACCESS_KEY": "ip3cdrkXcHmH4S7if7erKPNoxDn27V0vrg6CHHem",
+            "AWS_ACCESS_KEY_ID": "minioadmin",
+            "AWS_SECRET_ACCESS_KEY": "minioadmin",
             "PROM_URL":"http://localhost:9091/",
             "HOSTNAME":"test",
             "BACKUP_DIR": "/Users/simonbeck/Desktop/data",

.vscode/settings.json

@@ -1,9 +1,10 @@
 {
-  "go.testTags": "unit integration",
+  "go.testTags": "integration",
   "go.testTimeout": "120s",
   "go.testFlags": [
     "-v",
     "--race",
+    "--count=1",
   ],
   "go.testEnvVars": {
     "RESTIC_REPOSITORY":"s3:http://localhost:9000/baas1",
@@ -13,11 +14,11 @@
     "BACKUP_DIR":"testdata",
     "HOSTNAME":"test",
     "STATS_URL":"http://localhost:8091",
-    "PROM_URL": "http://localhost:8090",
+    // "PROM_URL": "http://localhost:8090",
     "RESTORE_ACCESSKEYID": "8U0UDNYPNUDTUS1LIAF3",
     "RESTORE_SECRETACCESSKEY":"ip3cdrkXcHmH4S7if7erKPNoxDn27V0vrg6CHHem",
     "RESTORE_S3ENDPOINT": "http://localhost:9000/restore",
     "RESTORE_DIR":"/tmp/restore",
-    "RESTIC_BINARY": "/Users/simonbeck/repos/restic/cmd/restic/restic"
+    "RESTIC_BINARY": "/usr/local/bin/restic"
   },
 }

CHANGELOG.md

@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Complete rewrite of whole wrestic
+- Better logging
+- Better code design
+- Better error handling
+- Better maintainability
+- Less deadlock potential with command IOs
 
 ## [v0.1.9] 2020-05-04
 ### Changed

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.13 as build
+FROM docker.io/golang:1.14 as build
 
 RUN set -x; \
     apt-get update \

cmd/wrestic/integration_test.go

@@ -17,10 +17,12 @@ import (
 	"testing"
 	"time"
 
-	"git.vshn.net/vshn/wrestic/s3"
+	"github.com/go-logr/glogr"
+	"github.com/go-logr/logr"
+	"github.com/vshn/wrestic/s3"
+	"github.com/vshn/wrestic/stats"
 
-	"git.vshn.net/vshn/wrestic/output"
-	"git.vshn.net/vshn/wrestic/restic"
+	"github.com/vshn/wrestic/restic"
 )
 
 type webhookserver struct {
@@ -39,11 +41,12 @@ func (s *testServer) Shutdown(ctx context.Context) {
 
 type testEnvironment struct {
 	s3Client  *s3.Client
-	output    *output.Output
 	webhook   *webhookserver
 	finishC   chan error
 	t         *testing.T
 	resticCli *restic.Restic
+	log       logr.Logger
+	stats     *stats.Handler
 }
 
 func assertOK(t *testing.T, err error) {
@@ -52,10 +55,6 @@ func assertOK(t *testing.T, err error) {
 	}
 }
 
-func newTestOutput() *output.Output {
-	return output.New(os.Getenv(webhookURLEnv), os.Getenv(promURLEnv), os.Getenv(restic.Hostname))
-}
-
 func newTestErrorChannel() chan error {
 	return make(chan error)
 }
@@ -83,15 +82,20 @@ func (w *webhookserver) runWebServer(t *testing.T) {
 	w.srv = srv
 }
 
+func getS3Repo() string {
+	resticString := os.Getenv("RESTIC_REPOSITORY")
+	resticString = strings.ToLower(resticString)
+
+	return strings.Replace(resticString, "s3:", "", -1)
+}
+
 func initTest(t *testing.T) *testEnvironment {
-	output := newTestOutput()
-	var dir string
-	if os.Getenv(restic.BackupDirEnv) == "" {
-		dir = "/data"
-	} else {
-		dir = os.Getenv(restic.BackupDirEnv)
-	}
-	resticCli := restic.New(dir, output)
+
+	mainLogger := glogr.New().WithName("wrestic")
+
+	statHandler := stats.NewHandler(os.Getenv(promURLEnv), os.Getenv(restic.Hostname), os.Getenv(webhookURLEnv), mainLogger)
+
+	resticCli := restic.New(context.TODO(), mainLogger, statHandler)
 
 	webhook := &webhookserver{}
 	webhook.runWebServer(t)
@@ -100,12 +104,13 @@ func initTest(t *testing.T) *testEnvironment {
 	s3client.DeleteBucket()
 	resetFlags()
 	return &testEnvironment{
-		output:    output,
 		finishC:   newTestErrorChannel(),
 		webhook:   webhook,
 		s3Client:  s3client,
 		t:         t,
 		resticCli: resticCli,
+		log:       mainLogger,
+		stats:     statHandler,
 	}
 }
 
@@ -125,11 +130,7 @@ func resetFlags() {
 func testBackup(t *testing.T) *testEnvironment {
 	env := initTest(t)
 
-	go run(env.finishC, env.output, env.resticCli)
-
-	assertOK(t, <-env.finishC)
-
-	env.output.TriggerAll()
+	assertOK(t, run(env.resticCli, env.log))
 
 	return env
 }
@@ -188,9 +189,7 @@ func TestRestore(t *testing.T) {
 	rstType := "s3"
 	restoreType = &rstType
 
-	go run(env.finishC, env.output, env.resticCli)
-
-	assertOK(t, <-env.finishC)
+	assertOK(t, run(env.resticCli, env.log))
 
 	env.webhook.srv.Shutdown(context.TODO())
 
@@ -201,7 +200,7 @@ func TestRestore(t *testing.T) {
 func TestBackup(t *testing.T) {
 	env := testBackup(t)
 
-	webhookData := restic.WebhookStats{}
+	webhookData := restic.BackupStats{}
 	assertOK(t, json.Unmarshal(env.webhook.jsonData, &webhookData))
 
 	if len(webhookData.Snapshots) != 2 {
@@ -222,9 +221,7 @@ func TestRestoreDisk(t *testing.T) {
 
 	os.Setenv("TRIM_RESTOREPATH", "false")
 
-	go run(env.finishC, env.output, env.resticCli)
-
-	assertOK(t, <-env.finishC)
+	assertOK(t, run(env.resticCli, env.log))
 
 	env.webhook.srv.Shutdown(context.TODO())
 
@@ -243,11 +240,9 @@ func TestInitRepoFail(t *testing.T) {
 	env := initTest(t)
 	defer testResetEnvVars(oldEnvVars)
 
-	go run(env.finishC, env.output, env.resticCli)
-
-	err := <-env.finishC
+	err := run(env.resticCli, env.log)
 
-	if err == nil || !strings.Contains(err.Error(), "connection refused") {
+	if err == nil || !strings.Contains(err.Error(), "exit status 1") {
 		t.Errorf("command did not fail with expected error, received error was: %v", err)
 	}
 
@@ -263,11 +258,7 @@ func TestArchive(t *testing.T) {
 	restoreTypeVar := "s3"
 	restoreType = &restoreTypeVar
 
-	go run(env.finishC, env.output, env.resticCli)
-
-	assertOK(t, <-env.finishC)
-
-	env.output.TriggerAll()
+	assertOK(t, run(env.resticCli, env.log))
 
 	env.webhook.srv.Shutdown(context.TODO())