Skip to content

Commit

Permalink
Merge pull request #63 from form3tech-oss/mike-logging-apply-chaos
Browse files Browse the repository at this point in the history
chore: Add log fields for apply chaos and surrounding code
  • Loading branch information
miketonks-form3 authored Dec 11, 2023
2 parents d7d3b7a + 8f0cfbf commit 2eb86b6
Showing 1 changed file with 13 additions and 11 deletions.
24 changes: 13 additions & 11 deletions controllers/common/records/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ const (
// Reconcile the chaos records
func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
obj := r.Object.DeepCopyObject().(v1alpha1.InnerObjectWithSelector)

if err := r.Client.Get(context.TODO(), req.NamespacedName, obj); err != nil {
if apierrors.IsNotFound(err) {
r.Log.Info("chaos not found")
Expand All @@ -79,11 +78,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
records := obj.GetStatus().Experiment.Records
selectors := obj.GetSelectorSpecs()

logger := r.Log.WithValues("name", obj.GetName(), "namespace", obj.GetNamespace(), "kind", obj.GetObjectKind().GroupVersionKind().Kind)

if records == nil {
for name, sel := range selectors {
targets, err := r.Selector.Select(context.TODO(), sel)
if err != nil {
r.Log.Error(err, "fail to select")
logger.Error(err, "fail to select")
r.Recorder.Event(obj, recorder.Failed{
Activity: "select targets",
Err: err.Error(),
Expand All @@ -92,7 +93,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
}

if len(targets) == 0 {
r.Log.Info("no target has been selected")
logger.Info("no target has been selected")
r.Recorder.Event(obj, recorder.Failed{
Activity: "select targets",
Err: "no target has been selected",
Expand All @@ -115,7 +116,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
needRetry := false
for index, record := range records {
var err error
r.Log.Info("iterating record", "record", record, "desiredPhase", desiredPhase)
idLogger := logger.WithValues("id", records[index].Id)
idLogger.Info("iterating record", "record", record, "desiredPhase", desiredPhase)

// The whole running logic is a cycle:
// Not Injected -> Not Injected/* -> Injected -> Injected/* -> Not Injected
Expand Down Expand Up @@ -146,15 +148,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
}

if operation == Apply {
r.Log.Info("apply chaos", "id", records[index].Id)
idLogger.Info("apply chaos")
record.Phase, err = r.Impl.Apply(context.TODO(), index, records, obj)
if record.Phase != originalPhase {
shouldUpdate = true
}
if err != nil {
// TODO: add backoff and retry mechanism
// but the retry shouldn't block other resource process
r.Log.Error(err, "fail to apply chaos")
idLogger.Error(err, "fail to apply chaos")
applyFailedEvent := newRecordEvent(v1alpha1.TypeFailed, v1alpha1.Apply, err.Error())
records[index].Events = append(records[index].Events, *applyFailedEvent)
r.Recorder.Event(obj, recorder.Failed{
Expand All @@ -176,15 +178,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
})
}
} else if operation == Recover {
r.Log.Info("recover chaos", "id", records[index].Id)
idLogger.Info("recover chaos")
record.Phase, err = r.Impl.Recover(context.TODO(), index, records, obj)
if record.Phase != originalPhase {
shouldUpdate = true
}
if err != nil {
// TODO: add backoff and retry mechanism
// but the retry shouldn't block other resource process
r.Log.Error(err, "fail to recover chaos")
idLogger.Error(err, "fail to recover chaos")
recoverFailedEvent := newRecordEvent(v1alpha1.TypeFailed, v1alpha1.Recover, err.Error())
records[index].Events = append(records[index].Events, *recoverFailedEvent)
r.Recorder.Event(obj, recorder.Failed{
Expand Down Expand Up @@ -215,11 +217,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
}
if shouldUpdate {
updateError := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
r.Log.Info("updating records", "records", records)
logger.Info("updating records", "records", records)
obj := r.Object.DeepCopyObject().(v1alpha1.InnerObjectWithSelector)

if err := r.Client.Get(context.TODO(), req.NamespacedName, obj); err != nil {
r.Log.Error(err, "unable to get chaos")
logger.Error(err, "unable to get chaos")
return err
}

Expand All @@ -232,7 +234,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
return r.Client.Update(context.TODO(), obj)
})
if updateError != nil {
r.Log.Error(updateError, "fail to update")
logger.Error(updateError, "fail to update")
r.Recorder.Event(obj, recorder.Failed{
Activity: "update records",
Err: updateError.Error(),
Expand Down

0 comments on commit 2eb86b6

Please sign in to comment.