From d03391437afbf14794535e7215a1ce51af129a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Thu, 27 May 2021 14:42:53 +0200 Subject: [PATCH 1/3] Fix backing up of annotated pods --- Makefile | 3 ++ Makefile.vars.mk | 2 + cmd/wrestic/main.go | 104 +++++++++++++++++++++++++------------------- 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index a3aa1521..a3ed0e65 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,9 @@ minio-download: .test/minio ## Download minio restic-download: .test/restic ## Download restic +docker: ## Builds the docker image. Overwrite the tag by changing the 'docker_tag' variable. + @docker build -t $(docker_tag) . + $(minio_pid): export MINIO_ACCESS_KEY = $(minio_root_user) $(minio_pid): export MINIO_SECRET_KEY = $(minio_root_password) $(minio_pid): minio-download diff --git a/Makefile.vars.mk b/Makefile.vars.mk index 84897c22..8d6746d3 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -31,3 +31,5 @@ minio_root_user ?= accesskey minio_root_password ?= secretkey minio_pid ?= $(minio_path).pid minio_url ?= https://dl.min.io/server/minio/release/$(os)-$(arch)/minio + +docker_tag ?= vshn/wrestic:snapshot diff --git a/cmd/wrestic/main.go b/cmd/wrestic/main.go index 27f1a648..ae36eff0 100644 --- a/cmd/wrestic/main.go +++ b/cmd/wrestic/main.go @@ -97,10 +97,10 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } } - commandRun := false + dontBackup := false if *prune { - commandRun = true + dontBackup = true if err := resticCLI.Prune(tags); err != nil { mainLogger.Error(err, "prune job failed") return err @@ -108,7 +108,7 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *check { - commandRun = true + dontBackup = true if err := resticCLI.Check(); err != nil { mainLogger.Error(err, "check job failed") return err @@ -116,7 +116,7 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *restore { - commandRun = true + dontBackup = true if err := resticCLI.Restore(*restoreSnap, restic.RestoreOptions{ RestoreType: restic.RestoreType(*restoreType), RestoreDir: os.Getenv(restic.RestoreDirEnv), @@ -134,59 +134,73 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *archive { - commandRun = true + dontBackup = true if err := resticCLI.Archive(*restoreFilter, *verifyRestore, tags); err != nil { mainLogger.Error(err, "archive job failed") return err } } - if !commandRun { - commandAnnotation := os.Getenv(commandEnv) - if commandAnnotation == "" { - commandAnnotation = "k8up.syn.tools/backupcommand" - } - fileextAnnotation := os.Getenv(fileextEnv) - if fileextAnnotation == "" { - fileextAnnotation = "k8up.syn.tools/file-extension" - } + if dontBackup { + return nil + } - _, serviceErr := os.Stat("/var/run/secrets/kubernetes.io") - _, kubeconfigErr := os.Stat(kubernetes.Kubeconfig) - - if serviceErr == nil && kubeconfigErr == nil { - - podLister := kubernetes.NewPodLister(commandAnnotation, fileextAnnotation, os.Getenv(restic.Hostname), mainLogger) - - podList, err := podLister.ListPods() - - if err == nil { - for _, pod := range podList { - data, err := kubernetes.PodExec(pod, mainLogger) - if err != nil { - mainLogger.Error(errors.New("error occured during data stream from k8s"), "pod execution was interrupted") - return err - } - filename := fmt.Sprintf("/%s-%s", os.Getenv(restic.Hostname), pod.ContainerName) - err = resticCLI.StdinBackup(data, filename, pod.FileExtension, tags) - if err != nil { - mainLogger.Error(err, "backup commands failed") - return err - } - } - mainLogger.Info("all pod commands have finished successfully") - } else { - mainLogger.Error(err, "could not list pods", "namespace", os.Getenv(restic.Hostname)) - } - } + err := backupAnnotatedPods(resticCLI, mainLogger) + if err != nil { + mainLogger.Error(err, "backup job failed", "step", "backup of annotated pods") + return err + } + mainLogger.Info("backups of annotated jobs have finished successfully") - err := resticCLI.Backup(getBackupDir(), tags) + err = resticCLI.Backup(getBackupDir(), tags) + if err != nil { + mainLogger.Error(err, "backup job failed", "step", "backup of dir failed", "dir", getBackupDir()) + return err + } + + return nil +} + +func backupAnnotatedPods(resticCLI *restic.Restic, mainLogger logr.Logger) error { + commandAnnotation := os.Getenv(commandEnv) + if commandAnnotation == "" { + commandAnnotation = "k8up.syn.tools/backupcommand" + } + fileextAnnotation := os.Getenv(fileextEnv) + if fileextAnnotation == "" { + fileextAnnotation = "k8up.syn.tools/file-extension" + } + + _, serviceErr := os.Stat("/var/run/secrets/kubernetes.io") + _, kubeconfigErr := os.Stat(kubernetes.Kubeconfig) + + if serviceErr != nil && kubeconfigErr != nil { + mainLogger.Info("No kubernetes credentials configured: Can't check for annotated Pods.") + return nil + } + + podLister := kubernetes.NewPodLister(commandAnnotation, fileextAnnotation, os.Getenv(restic.Hostname), mainLogger) + podList, err := podLister.ListPods() + + if err != nil { + mainLogger.Error(err, "could not list pods", "namespace", os.Getenv(restic.Hostname)) + return nil + } + + for _, pod := range podList { + data, err := kubernetes.PodExec(pod, mainLogger) if err != nil { - mainLogger.Error(err, "backup job failed") + mainLogger.Error(errors.New("error occurred during data stream from k8s"), "pod execution was interrupted") + return err + } + filename := fmt.Sprintf("/%s-%s", os.Getenv(restic.Hostname), pod.ContainerName) + err = resticCLI.StdinBackup(data, filename, pod.FileExtension, tags) + if err != nil { + mainLogger.Error(err, "backup commands failed") return err } - } + return nil } From fd8e651b12471305ac6b8fbc64ebe5048fc84fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20M=C3=A4der?= Date: Thu, 27 May 2021 15:22:31 +0200 Subject: [PATCH 2/3] Update Makefile.vars.mk Co-authored-by: Chris --- Makefile.vars.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.vars.mk b/Makefile.vars.mk index 8d6746d3..69020ba5 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -32,4 +32,4 @@ minio_root_password ?= secretkey minio_pid ?= $(minio_path).pid minio_url ?= https://dl.min.io/server/minio/release/$(os)-$(arch)/minio -docker_tag ?= vshn/wrestic:snapshot +docker_tag ?= local.dev/vshn/wrestic:snapshot From d6ec0ed03566f0253146757367ba539715982ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Thu, 27 May 2021 15:25:49 +0200 Subject: [PATCH 3/3] Add changelog entry --- CHANGELOG.md | 8 +++++++- cmd/wrestic/main.go | 12 ++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 886a8886..30be56ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v0.3.1] 2021-05-27 +### Fixed +- Fix skipping backup of annotated pods ([#93]) + ## [v0.3.0] 2021-05-26 ### Changed - Refactor Integration tests ([#90]) @@ -162,7 +166,8 @@ compatibility with older operator versions. Changes to the design contain: - Improved error detection and various bugfixes - Timeout for initial snapshot listing, default: 30s -[Unreleased]: https://git.vshn.net/vshn/wrestic/compare/v0.3.0...master +[Unreleased]: https://github.com/vshn/wrestic/compare/v0.3.0...master +[v0.3.1]: https://github.com/vshn/wrestic/releases/tag/v0.3.1 [v0.3.0]: https://github.com/vshn/wrestic/releases/tag/v0.3.0 [v0.0.7]: https://git.vshn.net/vshn/wrestic/compare/v0.0.6...v0.0.7 [v0.0.6]: https://git.vshn.net/vshn/wrestic/compare/v0.0.5...v0.0.6 @@ -187,3 +192,4 @@ compatibility with older operator versions. Changes to the design contain: [#89]: https://github.com/vshn/wrestic/pull/89 [#90]: https://github.com/vshn/wrestic/pull/90 [#92]: https://github.com/vshn/wrestic/pull/92 +[#93]: https://github.com/vshn/wrestic/pull/93 diff --git a/cmd/wrestic/main.go b/cmd/wrestic/main.go index ae36eff0..a841ffef 100644 --- a/cmd/wrestic/main.go +++ b/cmd/wrestic/main.go @@ -97,10 +97,10 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } } - dontBackup := false + skipBackup := false if *prune { - dontBackup = true + skipBackup = true if err := resticCLI.Prune(tags); err != nil { mainLogger.Error(err, "prune job failed") return err @@ -108,7 +108,7 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *check { - dontBackup = true + skipBackup = true if err := resticCLI.Check(); err != nil { mainLogger.Error(err, "check job failed") return err @@ -116,7 +116,7 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *restore { - dontBackup = true + skipBackup = true if err := resticCLI.Restore(*restoreSnap, restic.RestoreOptions{ RestoreType: restic.RestoreType(*restoreType), RestoreDir: os.Getenv(restic.RestoreDirEnv), @@ -134,14 +134,14 @@ func run(resticCLI *restic.Restic, mainLogger logr.Logger) error { } if *archive { - dontBackup = true + skipBackup = true if err := resticCLI.Archive(*restoreFilter, *verifyRestore, tags); err != nil { mainLogger.Error(err, "archive job failed") return err } } - if dontBackup { + if skipBackup { return nil }