Limit pod resources

This patch introduces back limits for the pods spawned by the test-operator after they were increased and later removed with these two PRs [1][2]. The problem with the previous two patches was that they only set the Resources.Limits field and not the Resources.Requests field. When Resources.Limits is set and Resources.Requests is empty then it inherrits the value from Resources.Limits. Therefore, we first hit the OOM killed issue when we set the Resources.Limits too low and later when we increased the value we hit the "Insufficient memory" error (due to high value in Resources.Requests field) This patch addresses the above mentioned issue by: - setting sane default values for Resource.Limits - setting sane default values for Resource.Requests and - introduces new parameter called .Spec.Resources which can be used to change the default values. [1] #222 [2] #224
openstack-k8s-operators · Oct 30, 2024 · 1f7805e · 1f7805e
1 parent eae3554
commit 1f7805e
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 0 deletions.
diff --git a/api/v1beta1/common.go b/api/v1beta1/common.go
@@ -95,6 +95,11 @@ type CommonOptions struct {
 	// This value contains a toleration that is applied to pods spawned by the
 	// test pods that are spawned by the test-operator.
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// Specify Resource Requirements for a each test pod spawned
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
 type CommonOpenstackConfig struct {
@@ -179,4 +184,9 @@ type WorkflowCommonParameters struct {
 	// This value contains a toleration that is applied to pods spawned by the
 	// test pods that are spawned by the test-operator.
 	Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// Specify Resource Requirements for a each test pod spawned
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
 }
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/controllers/ansibletest_controller.go b/controllers/ansibletest_controller.go
@@ -240,6 +240,10 @@ func (r *AnsibleTestReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		if instance.Spec.Workflow[externalWorkflowCounter].SELinuxLevel != nil {
 			instance.Spec.SELinuxLevel = *instance.Spec.Workflow[externalWorkflowCounter].SELinuxLevel
 		}
+
+		if instance.Spec.Workflow[externalWorkflowCounter].Resources != nil {
+			instance.Spec.Resources = *instance.Spec.Workflow[externalWorkflowCounter].Resources
+		}
 	}
 
 	// Service account, role, binding
@@ -263,6 +267,7 @@ func (r *AnsibleTestReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		externalWorkflowCounter,
 		containerImage,
 		privileged,
+		instance.Spec.Resources,
 	)
 	ansibleTestsJob := job.NewJob(
 		jobDef,

diff --git a/controllers/horizontest_controller.go b/controllers/horizontest_controller.go
@@ -230,6 +230,7 @@ func (r *HorizonTestReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		mountKubeconfig,
 		envVars,
 		containerImage,
+		instance.Spec.Resources,
 	)
 	horizontestJob := job.NewJob(
 		jobDef,

diff --git a/controllers/tempest_controller.go b/controllers/tempest_controller.go
@@ -331,6 +331,10 @@ func (r *TempestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 		if instance.Spec.Workflow[externalWorkflowCounter].SELinuxLevel != nil {
 			instance.Spec.SELinuxLevel = *instance.Spec.Workflow[externalWorkflowCounter].SELinuxLevel
 		}
+
+		if instance.Spec.Workflow[externalWorkflowCounter].Resources != nil {
+			instance.Spec.Resources = *instance.Spec.Workflow[externalWorkflowCounter].Resources
+		}
 	}
 
 	jobDef := tempest.Job(
@@ -344,6 +348,7 @@ func (r *TempestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 		mountCerts,
 		mountSSHKey,
 		containerImage,
+		instance.Spec.Resources,
 	)
 	tempestJob := job.NewJob(
 		jobDef,

diff --git a/controllers/tobiko_controller.go b/controllers/tobiko_controller.go
@@ -317,6 +317,7 @@ func (r *TobikoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 		envVars,
 		containerImage,
 		privileged,
+		instance.Spec.Resources,
 	)
 	tobikoJob := job.NewJob(
 		jobDef,
@@ -426,6 +427,10 @@ func (r *TobikoReconciler) PrepareTobikoEnvVars(
 		if instance.Spec.Workflow[step].SELinuxLevel != nil {
 			instance.Spec.SELinuxLevel = *instance.Spec.Workflow[step].SELinuxLevel
 		}
+
+		if instance.Spec.Workflow[step].Resources != nil {
+			instance.Spec.Resources = *instance.Spec.Workflow[step].Resources
+		}
 	}
 
 	// Prepare env vars

diff --git a/pkg/ansibletest/job.go b/pkg/ansibletest/job.go
@@ -7,6 +7,7 @@ import (
 	util "github.com/openstack-k8s-operators/test-operator/pkg/util"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -22,6 +23,7 @@ func Job(
 	externalWorkflowCounter int,
 	containerImage string,
 	privileged bool,
+	resources corev1.ResourceRequirements,
 ) *batchv1.Job {
 
 	runAsUser := int64(227)
@@ -32,6 +34,18 @@ func Job(
 	capabilities := []corev1.Capability{"NET_ADMIN", "NET_RAW"}
 	securityContext := util.GetSecurityContext(runAsUser, capabilities, privileged)
 
+	defaultResources := corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("4000Mi"),
+			corev1.ResourceMemory: resource.MustParse("10Gi"),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("2000Mi"),
+			corev1.ResourceMemory: resource.MustParse("5Gi"),
+		},
+	}
+	resources = util.GetResourceRequirements(resources, defaultResources)
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      jobName,
@@ -64,6 +78,7 @@ func Job(
 							Env:             env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:    GetVolumeMounts(mountCerts, instance, externalWorkflowCounter),
 							SecurityContext: &securityContext,
+							Resources:       resources,
 						},
 					},
 					Volumes: GetVolumes(

diff --git a/pkg/horizontest/job.go b/pkg/horizontest/job.go
@@ -7,6 +7,7 @@ import (
 	util "github.com/openstack-k8s-operators/test-operator/pkg/util"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -21,6 +22,7 @@ func Job(
 	mountKubeconfig bool,
 	envVars map[string]env.Setter,
 	containerImage string,
+	resources corev1.ResourceRequirements,
 ) *batchv1.Job {
 
 	runAsUser := int64(42455)
@@ -31,6 +33,18 @@ func Job(
 	capabilities := []corev1.Capability{"NET_ADMIN", "NET_RAW"}
 	securityContext := util.GetSecurityContext(runAsUser, capabilities, instance.Spec.Privileged)
 
+	defaultResources := corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("4000Mi"),
+			corev1.ResourceMemory: resource.MustParse("10Gi"),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("2000Mi"),
+			corev1.ResourceMemory: resource.MustParse("5Gi"),
+		},
+	}
+	resources = util.GetResourceRequirements(resources, defaultResources)
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      jobName,
@@ -63,6 +77,7 @@ func Job(
 							Env:             env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:    GetVolumeMounts(mountCerts, mountKeys, mountKubeconfig, instance),
 							SecurityContext: &securityContext,
+							Resources:       resources,
 						},
 					},
 					Volumes: GetVolumes(

diff --git a/pkg/tempest/job.go b/pkg/tempest/job.go
@@ -7,6 +7,7 @@ import (
 	util "github.com/openstack-k8s-operators/test-operator/pkg/util"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -22,13 +23,26 @@ func Job(
 	mountCerts bool,
 	mountSSHKey bool,
 	containerImage string,
+	resources corev1.ResourceRequirements,
 ) *batchv1.Job {
 
 	envVars := map[string]env.Setter{}
 	runAsUser := int64(42480)
 	runAsGroup := int64(42480)
 	securityContext := util.GetSecurityContext(runAsUser, []corev1.Capability{}, instance.Spec.Privileged)
 
+	defaultResources := corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("4000Mi"),
+			corev1.ResourceMemory: resource.MustParse("10Gi"),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("2000Mi"),
+			corev1.ResourceMemory: resource.MustParse("5Gi"),
+		},
+	}
+	resources = util.GetResourceRequirements(resources, defaultResources)
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      jobName,
@@ -60,6 +74,7 @@ func Job(
 							Env:             env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:    GetVolumeMounts(mountCerts, mountSSHKey, instance),
 							SecurityContext: &securityContext,
+							Resources:       resources,
 							EnvFrom: []corev1.EnvFromSource{
 								{
 									ConfigMapRef: &corev1.ConfigMapEnvSource{

diff --git a/pkg/tobiko/job.go b/pkg/tobiko/job.go
@@ -7,6 +7,7 @@ import (
 	util "github.com/openstack-k8s-operators/test-operator/pkg/util"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -23,6 +24,7 @@ func Job(
 	envVars map[string]env.Setter,
 	containerImage string,
 	privileged bool,
+	resources corev1.ResourceRequirements,
 ) *batchv1.Job {
 
 	runAsUser := int64(42495)
@@ -33,6 +35,18 @@ func Job(
 	capabilities := []corev1.Capability{"NET_ADMIN", "NET_RAW"}
 	securityContext := util.GetSecurityContext(runAsUser, capabilities, privileged)
 
+	defaultResources := corev1.ResourceRequirements{
+		Limits: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("4000Mi"),
+			corev1.ResourceMemory: resource.MustParse("10Gi"),
+		},
+		Requests: corev1.ResourceList{
+			corev1.ResourceCPU:    resource.MustParse("2000Mi"),
+			corev1.ResourceMemory: resource.MustParse("5Gi"),
+		},
+	}
+	resources = util.GetResourceRequirements(resources, defaultResources)
+
 	// Note(lpiwowar): Once the webhook is implemented move all the logic of merging
 	//                 the workflows there.
 	job := &batchv1.Job{
@@ -68,6 +82,7 @@ func Job(
 							Env:             env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:    GetVolumeMounts(mountCerts, mountKeys, mountKubeconfig, instance),
 							SecurityContext: &securityContext,
+							Resources:       resources,
 						},
 					},
 					Volumes: GetVolumes(

diff --git a/pkg/util/common.go b/pkg/util/common.go
@@ -36,3 +36,22 @@ func GetSecurityContext(
 
 	return securityContext
 }
+
+func GetResourceRequirements(
+	resourceRequirements corev1.ResourceRequirements,
+	defaultResourceRequirements corev1.ResourceRequirements,
+) corev1.ResourceRequirements {
+	if resourceRequirements.Limits == nil {
+		resourceRequirements.Limits = defaultResourceRequirements.Limits
+	}
+
+	if resourceRequirements.Requests == nil {
+		resourceRequirements.Requests = defaultResourceRequirements.Requests
+	}
+
+	if resourceRequirements.Claims == nil {
+		resourceRequirements.Claims = defaultResourceRequirements.Claims
+	}
+
+	return resourceRequirements
+}