From e86812cee99f394ffa45a3e4be5c47fc45fe1133 Mon Sep 17 00:00:00 2001 From: Mike Baum Date: Mon, 6 May 2024 09:03:08 -0400 Subject: [PATCH] Add env property to pipeline API to allow setting env vars in the pipeline pod Signed-off-by: Mike Baum --- README.md | 28 +++-- api/v1alpha1/pipeline_types.go | 4 + api/v1alpha1/zz_generated.deepcopy.go | 8 ++ .../bases/captain.benthos.dev_pipelines.yaml | 115 +++++++++++++++++- config/samples/captain_v1alpha1_pipeline.yaml | 26 ++-- internal/pkg/resource/resource.go | 1 + 6 files changed, 162 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c36fb03..0d9a3df 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,19 @@ kind: Pipeline metadata: name: pipeline-sample spec: - replicas: 2 + replicas: 1 config: input: - broker: - inputs: - - file: - paths: ["./config/meow.txt"] - - generate: - mapping: root = "woof" - interval: 10s - count: 0 + generate: + mapping: | + let favorite_animal = env("FAVORITE_ANIMAL") + root = match $favorite_animal { + "cat" => file("/config/cat.txt") + "dog" => file("/config/dog.txt") + _ => file("/config/dog.txt") + } + interval: 5s + count: 0 pipeline: processors: @@ -39,8 +41,14 @@ spec: stdout: {} configFiles: - meow.txt: | + cat.txt: | meow + dog.txt: | + woof + + env: + - name: FAVORITE_ANIMAL + value: cat ``` Once the resource is deployed, you can monitor the state of the resource: diff --git a/api/v1alpha1/pipeline_types.go b/api/v1alpha1/pipeline_types.go index f66762f..a2964d1 100644 --- a/api/v1alpha1/pipeline_types.go +++ b/api/v1alpha1/pipeline_types.go @@ -1,6 +1,7 @@ package v1alpha1 import ( + v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -29,6 +30,9 @@ type PipelineSpec struct { // ConfigFiles Additional configuration, as Key/Value pairs, that will be mounted as files with the /config // directory on the pod. The key should be the file name and the value should be its content. ConfigFiles map[string]string `json:"configFiles,omitempty"` + + // Env Environment Variables to set in the benthos pipeline pod. + Env []v1.EnvVar `json:"env,omitempty"` } // PipelineStatus defines the observed state of Pipeline diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 728767b..037b134 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5,6 +5,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -83,6 +84,13 @@ func (in *PipelineSpec) DeepCopyInto(out *PipelineSpec) { (*out)[key] = val } } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelineSpec. diff --git a/config/crd/bases/captain.benthos.dev_pipelines.yaml b/config/crd/bases/captain.benthos.dev_pipelines.yaml index 2cf046d..745805f 100644 --- a/config/crd/bases/captain.benthos.dev_pipelines.yaml +++ b/config/crd/bases/captain.benthos.dev_pipelines.yaml @@ -68,8 +68,121 @@ spec: type: string description: |- ConfigFiles Additional configuration, as Key/Value pairs, that will be mounted as files with the /config - directory on the pod. The key should be the file name and the value should be it's content. + directory on the pod. The key should be the file name and the value should be its content. type: object + env: + description: Env Environment Variable to set in the benthos pipeline + pod. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array image: description: Image defines the image and tag to use for the Benthos deployment. diff --git a/config/samples/captain_v1alpha1_pipeline.yaml b/config/samples/captain_v1alpha1_pipeline.yaml index c1dab07..a70a595 100644 --- a/config/samples/captain_v1alpha1_pipeline.yaml +++ b/config/samples/captain_v1alpha1_pipeline.yaml @@ -12,14 +12,16 @@ spec: replicas: 1 config: input: - broker: - inputs: - - file: - paths: ["./config/meow.txt"] - - generate: - mapping: root = "woof" - interval: 10s - count: 0 + generate: + mapping: | + let favorite_animal = env("FAVORITE_ANIMAL") + root = match $favorite_animal { + "cat" => file("/config/cat.txt") + "dog" => file("/config/dog.txt") + _ => file("/config/dog.txt") + } + interval: 5s + count: 0 pipeline: processors: @@ -29,5 +31,11 @@ spec: stdout: {} configFiles: - meow.txt: | + cat.txt: | meow + dog.txt: | + woof + + env: + - name: FAVORITE_ANIMAL + value: cat diff --git a/internal/pkg/resource/resource.go b/internal/pkg/resource/resource.go index c0f1766..cbaba9b 100644 --- a/internal/pkg/resource/resource.go +++ b/internal/pkg/resource/resource.go @@ -54,6 +54,7 @@ func NewDeployment(name string, namespace string, scope captainv1.PipelineSpec) ReadOnly: true, }, }, + Env: scope.Env, }}, Volumes: []corev1.Volume{ {