fix: don't spawn new workflow child on abort

If status check node aborted and has "abort with statuscheck" flag set, don't spawn a new workflow child node for the next step. This fixes a race condition where the serial reconciler spawns a new child node in the window between the statuscheck node aborting and the abort status being propagated to the parent. Signed-off-by: Graham Brereton <[email protected]>
form3tech-oss · Dec 15, 2023 · 9bf5d79 · 9bf5d79
1 parent 374536a
commit 9bf5d79
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -53,6 +53,7 @@ For more information and how-to, see [RFC: Keep A Changelog](https://github.com/
 - Fix goroutine leak [#4229](https://github.com/chaos-mesh/chaos-mesh/pull/4229)
 - Remove the duplicate `make test` [#4234](https://github.com/chaos-mesh/chaos-mesh/pull/4234)
 - Fix daemon-server `SetDNSServer` endpoint to validate provided server address [#4246](https://github.com/chaos-mesh/chaos-mesh/pull/4246)
+- Fix serial workflow node reconciler not to spawn the next child after status check abort [#4286](https://github.com/chaos-mesh/chaos-mesh/pull/4286)
 
 ### Security
 

diff --git a/pkg/workflow/controllers/serial_node_reconciler.go b/pkg/workflow/controllers/serial_node_reconciler.go
@@ -178,6 +178,16 @@ func (it *SerialNodeReconciler) syncChildNodes(ctx context.Context, node v1alpha
 	if err != nil {
 		return err
 	}
+
+	if abortedStatusCheck := findAbortedStatusCheckNode(finishedChildNodes); abortedStatusCheck != nil {
+		it.logger.Info(
+			"not spawning new child, status check node with status aborted and AbortWithStatusCheck = true",
+			"node",
+			fmt.Sprintf("%s/%s", abortedStatusCheck.Namespace, abortedStatusCheck.Name),
+		)
+		return nil
+	}
+
 	var taskToStartup string
 	if len(activeChildNodes) == 0 {
 		// no active children, trying to spawn a new one
@@ -287,3 +297,16 @@ func (it *SerialNodeReconciler) syncChildNodes(ctx context.Context, node v1alpha
 
 	return nil
 }
+
+func findAbortedStatusCheckNode(nodes []v1alpha1.WorkflowNode) *v1alpha1.WorkflowNode {
+	for _, node := range nodes {
+		if node.Spec.Type == v1alpha1.TypeStatusCheck && node.Spec.AbortWithStatusCheck {
+			for _, cond := range node.Status.Conditions {
+				if cond.Type == v1alpha1.ConditionAborted && cond.Status == corev1.ConditionTrue {
+					return &node
+				}
+			}
+		}
+	}
+	return nil
+}