From 34deecb9ff66706760d2143c849b0436071c061b Mon Sep 17 00:00:00 2001 From: afazekas Date: Wed, 20 Sep 2023 11:39:00 +0200 Subject: [PATCH 1/2] Remove TempestRegexp argument Looks like it was not in use since the begning. Users usually finds simpler to have explicit negative list, even tough it is possible to consutuct a singe regexp. Also possible to construct a single regexp on the operator side, in case other way of passing would be prefered later on. Moving the default allowedTests to the defaults. Note: Consider moving the tempest run arguments to sub structure. --- api/bases/test.openstack.org_tempests.yaml | 5 ++--- api/v1beta1/tempest_types.go | 5 +---- config/crd/bases/test.openstack.org_tempests.yaml | 5 ++--- pkg/tempest/job.go | 14 +++----------- templates/tempest/config/include.txt | 2 -- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/api/bases/test.openstack.org_tempests.yaml b/api/bases/test.openstack.org_tempests.yaml index 5c0e6dee..8b44cec2 100644 --- a/api/bases/test.openstack.org_tempests.yaml +++ b/api/bases/test.openstack.org_tempests.yaml @@ -36,6 +36,8 @@ spec: description: TempestSpec defines the desired state of Tempest properties: allowedTests: + default: + - tempest.api.identity.v3 description: AllowedTests items: type: string @@ -119,9 +121,6 @@ spec: items: type: string type: array - tempestRegex: - description: TempestRegex - type: string required: - containerImage - openStackConfigMap diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index 31687636..5fa577fd 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -64,10 +64,7 @@ type TempestSpec struct { ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` // +kubebuilder:validation:Optional - // TempestRegex - TempestRegex string `json:"tempestRegex,omitempty"` - - // +kubebuilder:validation:Optional + // +kubebuilder:default={"tempest.api.identity.v3"} // AllowedTests AllowedTests []string `json:"allowedTests,omitempty"` diff --git a/config/crd/bases/test.openstack.org_tempests.yaml b/config/crd/bases/test.openstack.org_tempests.yaml index 5c0e6dee..8b44cec2 100644 --- a/config/crd/bases/test.openstack.org_tempests.yaml +++ b/config/crd/bases/test.openstack.org_tempests.yaml @@ -36,6 +36,8 @@ spec: description: TempestSpec defines the desired state of Tempest properties: allowedTests: + default: + - tempest.api.identity.v3 description: AllowedTests items: type: string @@ -119,9 +121,6 @@ spec: items: type: string type: array - tempestRegex: - description: TempestRegex - type: string required: - containerImage - openStackConfigMap diff --git a/pkg/tempest/job.go b/pkg/tempest/job.go index 8b2b114e..6117c98e 100644 --- a/pkg/tempest/job.go +++ b/pkg/tempest/job.go @@ -19,14 +19,6 @@ func Job( runAsUser := int64(42480) runAsGroup := int64(42480) - args := []string{ - "/var/lib/tempest/run_tempest.sh", - } - if instance.Spec.TempestRegex != "" { - args = append(args, "--regex") - args = append(args, instance.Spec.TempestRegex) - } - job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -40,9 +32,9 @@ func Job( RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: instance.RbacResourceName(), SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &runAsUser, + RunAsUser: &runAsUser, RunAsGroup: &runAsGroup, - FSGroup: &runAsGroup, + FSGroup: &runAsGroup, }, Containers: []corev1.Container{ { @@ -51,7 +43,7 @@ func Job( Command: []string{ "/usr/local/bin/container-scripts/invoke_tempest", }, - Args: []string{}, + Args: []string{}, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: GetVolumeMounts(), }, diff --git a/templates/tempest/config/include.txt b/templates/tempest/config/include.txt index c36fd885..3a4f8979 100644 --- a/templates/tempest/config/include.txt +++ b/templates/tempest/config/include.txt @@ -1,5 +1,3 @@ {{range $val := .AllowedTests }} {{$val}} -{{ else }} -tempest.api.identity.v3 {{end}} From eb5dd4637304842a5b40024a342476dd5c4ce58b Mon Sep 17 00:00:00 2001 From: afazekas Date: Wed, 20 Sep 2023 15:34:05 +0200 Subject: [PATCH 2/2] Split tempest run args !backward incomatible! Moving the tempest run related args to their own section. Adding concurrency related args. --- api/bases/test.openstack.org_tempests.yaml | 36 ++++++++++----- api/v1beta1/tempest_types.go | 34 ++++++++++---- api/v1beta1/zz_generated.deepcopy.go | 45 ++++++++++++++----- .../bases/test.openstack.org_tempests.yaml | 36 ++++++++++----- config/samples/test_v1beta1_tempest.yaml | 5 ++- controllers/tempest_controller.go | 10 ++++- pkg/tempest/job.go | 6 +++ templates/tempest/bin/invoke_tempest | 9 +++- 8 files changed, 133 insertions(+), 48 deletions(-) diff --git a/api/bases/test.openstack.org_tempests.yaml b/api/bases/test.openstack.org_tempests.yaml index 8b44cec2..516f5582 100644 --- a/api/bases/test.openstack.org_tempests.yaml +++ b/api/bases/test.openstack.org_tempests.yaml @@ -35,13 +35,6 @@ spec: spec: description: TempestSpec defines the desired state of Tempest properties: - allowedTests: - default: - - tempest.api.identity.v3 - description: AllowedTests - items: - type: string - type: array backoffLimit: default: 0 description: BackoffLimimt allows to define the maximum number of @@ -116,11 +109,30 @@ spec: description: OpenStackConfigSecret is the name of the Secret containing the secure.yaml type: string - skippedTests: - description: SkippedTests - items: - type: string - type: array + tempestRun: + description: TempestSpec TempestRun parts + properties: + allowedTests: + default: + - tempest.api.identity.v3 + description: AllowedTests + items: + type: string + type: array + concurrency: + default: 0 + description: Concurrency is the Default concurrency + format: int64 + type: integer + skippedTests: + description: SkippedTests + items: + type: string + type: array + workerFile: + description: WorkerFile is the detailed concurry spec file + type: string + type: object required: - containerImage - openStackConfigMap diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index 5fa577fd..86f5213c 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -35,6 +35,27 @@ type Hash struct { Hash string `json:"hash,omitempty"` } +// TempestSpec TempestRun parts +type TempestRunSpec struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default={"tempest.api.identity.v3"} + // AllowedTests + AllowedTests []string `json:"allowedTests,omitempty"` + + // +kubebuilder:validation:Optional + // SkippedTests + SkippedTests []string `json:"skippedTests,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default:=0 + // Concurrency is the Default concurrency + Concurrency *int64 `json:"concurrency,omitempty"` + + // +kubebuilder:validation:Optional + // WorkerFile is the detailed concurry spec file + WorkerFile string `json:"workerFile,omitempty"` +} + // TempestSpec defines the desired state of Tempest type TempestSpec struct { // +kubebuilder:validation:Required @@ -63,23 +84,18 @@ type TempestSpec struct { // ExternalEndpoints, expose a VIP using a pre-created IPAddressPool ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` - // +kubebuilder:validation:Optional - // +kubebuilder:default={"tempest.api.identity.v3"} - // AllowedTests - AllowedTests []string `json:"allowedTests,omitempty"` - - // +kubebuilder:validation:Optional - // SkippedTests - SkippedTests []string `json:"skippedTests,omitempty"` - // BackoffLimimt allows to define the maximum number of retried executions (defaults to 6). // +kubebuilder:default:=0 // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:number"} BackoffLimit *int32 `json:"backoffLimit,omitempty"` + // +kubebuilder:validation:Optional + TempestRun *TempestRunSpec `json:"tempestRun,omitempty"` + // TODO(slaweq): add more tempest run parameters here } + // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { // +kubebuilder:validation:Required diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7b33c546..3f631537 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -120,6 +120,36 @@ func (in *TempestList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TempestRunSpec) DeepCopyInto(out *TempestRunSpec) { + *out = *in + if in.AllowedTests != nil { + in, out := &in.AllowedTests, &out.AllowedTests + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SkippedTests != nil { + in, out := &in.SkippedTests, &out.SkippedTests + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Concurrency != nil { + in, out := &in.Concurrency, &out.Concurrency + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestRunSpec. +func (in *TempestRunSpec) DeepCopy() *TempestRunSpec { + if in == nil { + return nil + } + out := new(TempestRunSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TempestSpec) DeepCopyInto(out *TempestSpec) { *out = *in @@ -142,21 +172,16 @@ func (in *TempestSpec) DeepCopyInto(out *TempestSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.AllowedTests != nil { - in, out := &in.AllowedTests, &out.AllowedTests - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.SkippedTests != nil { - in, out := &in.SkippedTests, &out.SkippedTests - *out = make([]string, len(*in)) - copy(*out, *in) - } if in.BackoffLimit != nil { in, out := &in.BackoffLimit, &out.BackoffLimit *out = new(int32) **out = **in } + if in.TempestRun != nil { + in, out := &in.TempestRun, &out.TempestRun + *out = new(TempestRunSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TempestSpec. diff --git a/config/crd/bases/test.openstack.org_tempests.yaml b/config/crd/bases/test.openstack.org_tempests.yaml index 8b44cec2..516f5582 100644 --- a/config/crd/bases/test.openstack.org_tempests.yaml +++ b/config/crd/bases/test.openstack.org_tempests.yaml @@ -35,13 +35,6 @@ spec: spec: description: TempestSpec defines the desired state of Tempest properties: - allowedTests: - default: - - tempest.api.identity.v3 - description: AllowedTests - items: - type: string - type: array backoffLimit: default: 0 description: BackoffLimimt allows to define the maximum number of @@ -116,11 +109,30 @@ spec: description: OpenStackConfigSecret is the name of the Secret containing the secure.yaml type: string - skippedTests: - description: SkippedTests - items: - type: string - type: array + tempestRun: + description: TempestSpec TempestRun parts + properties: + allowedTests: + default: + - tempest.api.identity.v3 + description: AllowedTests + items: + type: string + type: array + concurrency: + default: 0 + description: Concurrency is the Default concurrency + format: int64 + type: integer + skippedTests: + description: SkippedTests + items: + type: string + type: array + workerFile: + description: WorkerFile is the detailed concurry spec file + type: string + type: object required: - containerImage - openStackConfigMap diff --git a/config/samples/test_v1beta1_tempest.yaml b/config/samples/test_v1beta1_tempest.yaml index 95752674..d2ee6e52 100644 --- a/config/samples/test_v1beta1_tempest.yaml +++ b/config/samples/test_v1beta1_tempest.yaml @@ -5,4 +5,7 @@ metadata: namespace: openstack spec: containerImage: quay.io/podified-antelope-centos9/openstack-tempest:current-podified - secret: tempest-secret + tempestRun: + allowedTests: + - tempest.api.identity.v3.* + concurrency: 8 diff --git a/controllers/tempest_controller.go b/controllers/tempest_controller.go index 9bbc23fc..e1b47367 100644 --- a/controllers/tempest_controller.go +++ b/controllers/tempest_controller.go @@ -385,9 +385,14 @@ func (r *TempestReconciler) generateServiceConfigMaps( cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(tempest.ServiceName), map[string]string{}) templateParameters := make(map[string]interface{}) + customData := make(map[string]string) - templateParameters["AllowedTests"] = instance.Spec.AllowedTests - templateParameters["SkippedTests"] = instance.Spec.SkippedTests + if len(instance.Spec.TempestRun.WorkerFile) != 0 { + customData["worker_file.yaml"] = instance.Spec.TempestRun.WorkerFile + } + + templateParameters["AllowedTests"] = instance.Spec.TempestRun.AllowedTests + templateParameters["SkippedTests"] = instance.Spec.TempestRun.SkippedTests cms := []util.Template{ // ScriptsConfigMap @@ -406,6 +411,7 @@ func (r *TempestReconciler) generateServiceConfigMaps( InstanceType: instance.Kind, Labels: cmLabels, ConfigOptions: templateParameters, + CustomData: customData, }, } return configmap.EnsureConfigMaps(ctx, h, instance, cms, nil) diff --git a/pkg/tempest/job.go b/pkg/tempest/job.go index 6117c98e..a5ecf5de 100644 --- a/pkg/tempest/job.go +++ b/pkg/tempest/job.go @@ -7,6 +7,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strconv" ) // Job - prepare job to run Tempest tests @@ -18,6 +19,11 @@ func Job( envVars := map[string]env.Setter{} runAsUser := int64(42480) runAsGroup := int64(42480) + if instance.Spec.TempestRun.Concurrency != nil { + envVars["TEMPEST_CONCURRENCY"] = env.SetValue(strconv.FormatInt(*instance.Spec.TempestRun.Concurrency, 10)) + } else { + envVars["TEMPEST_CONCURRENCY"] = env.SetValue("0") + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ diff --git a/templates/tempest/bin/invoke_tempest b/templates/tempest/bin/invoke_tempest index 2ba88612..bb22f5a0 100644 --- a/templates/tempest/bin/invoke_tempest +++ b/templates/tempest/bin/invoke_tempest @@ -18,9 +18,14 @@ cd "$TEMPEST_DIR" # TODO: consider profile support, deployer-input and other args passing discover-tempest-config --os-cloud "$OS_CLOUD" --debug --create identity.v3_endpoint_type public +if [ -e $OPERATOR_ETC/worker_file.yaml ]; then + WORKER_FILE="--worker-file $OPERATOR_ETC/worker_file.yaml" +fi + tempest run \ - --include-list ${OPERATOR_ETC}/include.txt \ - --exclude-list ${OPERATOR_ETC}/exclude.txt + --concurrency "${TEMPEST_CONCURRENCY}" $WORKER_FILE \ + --include-list "${OPERATOR_ETC}/include.txt" \ + --exclude-list "${OPERATOR_ETC}/exclude.txt" # NOTE: We might not want to fail when tests are executed RETURN_VALUE=$?