Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

K8schaos update fix #62

Merged
merged 4 commits into from
Nov 29, 2023
Merged

K8schaos update fix #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d6e96a7
work in progress on k8schaos saving copy of resource in the status field
wvdschel-f3 Nov 28, 2023
a120de3
fix: k8schaos update resource verified to work in kind
wvdschel-f3 Nov 29, 2023
b068ae0
chore: remove some unnecessary logs from k8schaos
wvdschel-f3 Nov 29, 2023
35861f9
fix: CI error
wvdschel-f3 Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion api/v1alpha1/k8schaos_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,16 @@ type K8SChaosAPIObjects struct {

// K8SChaosStatus defines the observed state of K8SChaos
type K8SChaosStatus struct {
ChaosStatus `json:",inline"`
ChaosStatus `json:",inline"`
OriginalObjectValue string `json:"originalObjectValue,omitempty"`
}

func (obj *K8SChaos) GetSelectorSpecs() map[string]interface{} {
return map[string]interface{}{
".": obj.Spec.APIObjects,
}
}

func (obj *K8SChaos) GetCustomStatus() interface{} {
return &obj.Status.OriginalObjectValue
}
2 changes: 2 additions & 0 deletions config/crd/bases/chaos-mesh.org_k8schaos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ spec:
- Stop
type: string
type: object
originalObjectValue:
type: string
required:
- experiment
type: object
Expand Down
114 changes: 84 additions & 30 deletions controllers/chaosimpl/k8schaos/impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package k8schaos

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
Expand All @@ -40,8 +41,6 @@ var _ impltypes.ChaosImpl = (*Impl)(nil)
type Impl struct {
client.Client
Log logr.Logger

initialValue *unstructured.Unstructured
}

const (
Expand All @@ -52,12 +51,9 @@ const (
func (impl *Impl) Apply(ctx context.Context, index int, records []*v1alpha1.Record, obj v1alpha1.InnerObject) (v1alpha1.Phase, error) {
impl.Log.Info("k8schaos Apply", "namespace", obj.GetNamespace(), "name", obj.GetName())

// TODO: We need to consider the case where we're applying an object that already exists (updating it).
// In that case we should store the original objects in k8schaos.Status.OriginalObjects.
k8schaos, ok := obj.(*v1alpha1.K8SChaos)
if !ok {
err := errors.New("chaos is not K8SChaos")
impl.Log.Error(err, "chaos is not K8SChaos", "chaos", obj)
return v1alpha1.NotInjected, err
}

Expand Down Expand Up @@ -85,33 +81,57 @@ func (impl *Impl) Apply(ctx context.Context, index int, records []*v1alpha1.Reco
return v1alpha1.NotInjected, fmt.Errorf("get rest mapping: %w", err)
}

if k8schaos.Spec.Update {
impl.initialValue, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Get(ctx, resource.GetName(), v1.GetOptions{})
if err != nil && !apiErrors.IsNotFound(err) {
return v1alpha1.NotInjected, err
}
originalValue, err := client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Get(ctx, resource.GetName(), v1.GetOptions{})
if err != nil && !apiErrors.IsNotFound(err) {
return v1alpha1.NotInjected, err
}

if impl.initialValue != nil {
var resourceVersion string
var found bool
resourceVersion, found, err = unstructured.NestedString(impl.initialValue.Object, "metadata", "resourceVersion")
if err != nil {
return v1alpha1.NotInjected, err
}
if found {
resource.SetResourceVersion(resourceVersion)
}
if k8schaos.Spec.Update && originalValue != nil {
var resourceVersion string
var found bool
resourceVersion, found, err = unstructured.NestedString(originalValue.Object, "metadata", "resourceVersion")
if err != nil {
return v1alpha1.NotInjected, fmt.Errorf("resourceVersion is not a string: %w", err)
}
if found {
resource.SetResourceVersion(resourceVersion)
}

unstructured.RemoveNestedField(impl.initialValue.Object, "metadata", "creationTimestamp")
unstructured.RemoveNestedField(impl.initialValue.Object, "metadata", "resourceVersion")
unstructured.RemoveNestedField(impl.initialValue.Object, "metadata", "uid")
unstructured.RemoveNestedField(originalValue.Object, "metadata", "resourceVersion")
unstructured.RemoveNestedField(originalValue.Object, "metadata", "uid")
serialized, err := json.Marshal(originalValue)
if err != nil {
return v1alpha1.NotInjected, fmt.Errorf("failed to serialize original resource value: %w", err)
}

k8schaos.Status.OriginalObjectValue = string(serialized)

impl.Log.Info("k8schaos: updating existing resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "PUT", "resourceVersion", resourceVersion)
_, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Update(ctx, resource, v1.UpdateOptions{})
} else {
_, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Create(ctx, resource, v1.CreateOptions{})

if err != nil {
impl.Log.Error(err, "k8schaos: failed to update resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName())
return v1alpha1.NotInjected, err
}

return v1alpha1.Injected, nil
}

if k8schaos.Spec.Update {
impl.Log.Info("k8schaos: warning: chaos has update=true but resource not found - creating a new resource instead", "namespace",
obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName())
}

impl.Log.Info("k8schaos: creating new resources", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "POST")
_, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Create(ctx, resource, v1.CreateOptions{})

if err != nil {
impl.Log.Error(err, "k8schaos: failed to create resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName())
return v1alpha1.NotInjected, err
}

Expand All @@ -126,7 +146,6 @@ func (impl *Impl) Recover(ctx context.Context, index int, records []*v1alpha1.Re
k8schaos, ok := obj.(*v1alpha1.K8SChaos)
if !ok {
err := errors.New("chaos is not K8SChaos")
impl.Log.Error(err, "chaos is not K8SChaos", "chaos", obj)
return v1alpha1.Injected, err
}

Expand Down Expand Up @@ -160,12 +179,47 @@ func (impl *Impl) Recover(ctx context.Context, index int, records []*v1alpha1.Re
return v1alpha1.Injected, fmt.Errorf("resource is not managed by %s: %s: \"%s\"", managedBy, managedByLabel, resMgr)
}

if impl.initialValue != nil {
_, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Update(ctx, impl.initialValue, v1.UpdateOptions{})
} else {
err = resourceClient.Delete(ctx, resource.GetName(), v1.DeleteOptions{})
if k8schaos.Status.OriginalObjectValue != "" {
var recoveryValue unstructured.Unstructured
if err := json.Unmarshal([]byte(k8schaos.Status.OriginalObjectValue), &recoveryValue); err != nil {
return v1alpha1.Injected, fmt.Errorf("failed to load value to roll back to from status: %w", err)
}

var resourceVersion string
var found bool
resourceVersion, found, err = unstructured.NestedString(existingResource.Object, "metadata", "resourceVersion")
if err != nil {
return v1alpha1.Injected, fmt.Errorf("resourceVersion is not a string: %w", err)
}
if found {
recoveryValue.SetResourceVersion(resourceVersion)
}

impl.Log.Info("k8schaos: rolling back resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "PUT")
_, err = client.Resource(mapping.Resource).Namespace(resource.GetNamespace()).Update(ctx, &recoveryValue, v1.UpdateOptions{})
if err != nil {
impl.Log.Info("k8schaos: failed to roll back resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "PUT")
return v1alpha1.Injected, err
}

return v1alpha1.NotInjected, nil
}

if k8schaos.Spec.Update {
impl.Log.Info("k8schaos: warning: chaos has update=true but no resource is stored in status - resource will be deleted", "namespace",
obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName())
}

impl.Log.Info("k8schaos: deleting resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "DELETE")
err = resourceClient.Delete(ctx, resource.GetName(), v1.DeleteOptions{})

if err != nil && !apiErrors.IsNotFound(err) {
impl.Log.Info("k8schaos: failed to delete resource", "namespace", obj.GetNamespace(), "name", obj.GetName(),
"target-namespace", resource.GetNamespace(), "target-name", resource.GetName(), "method", "DELETE")
return v1alpha1.Injected, err
}

Expand Down
2 changes: 2 additions & 0 deletions helm/chaos-mesh/crds/chaos-mesh.org_k8schaos.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ spec:
- Stop
type: string
type: object
originalObjectValue:
type: string
required:
- experiment
type: object
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3008,6 +3008,8 @@ spec:
- Stop
type: string
type: object
originalObjectValue:
type: string
required:
- experiment
type: object
Expand Down
Loading