-
Notifications
You must be signed in to change notification settings - Fork 440
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create ServiceMonitor for operator metrics programmatically (#3371)
* Create ServiceMonitor for operator metrics programmatically Signed-off-by: Israel Blancas <[email protected]> * Apply changes requested in CR Signed-off-by: Israel Blancas <[email protected]> * Apply changes requested in CR Signed-off-by: Israel Blancas <[email protected]> --------- Signed-off-by: Israel Blancas <[email protected]>
- Loading branch information
Showing
17 changed files
with
411 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| # One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' | ||
| change_type: enhancement | ||
|
|
||
| # The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action) | ||
| component: operator | ||
|
|
||
| # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). | ||
| note: Programmatically create the `ServiceMonitor` for the operator metrics endpoint, ensuring correct namespace handling and dynamic configuration. | ||
|
|
||
| # One or more tracking issues related to the change | ||
| issues: [3370] | ||
|
|
||
| # (Optional) One or more lines of additional information to render under the primary note. | ||
| # These lines will be padded with 2 spaces and then inserted directly into the document. | ||
| # Use pipe (|) for multiline entries. | ||
| subtext: | | ||
| Previously, the `ServiceMonitor` was created statically from a manifest file, causing failures when the | ||
| operator was deployed in a non-default namespace. This enhancement ensures automatic adjustment of the | ||
| `serverName` and seamless metrics scraping. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 2 additions & 0 deletions
2
...shift/manifests/opentelemetry-operator-controller-manager-metrics-service_v1_service.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
...ests/opentelemetry-operator-prometheus-rules_monitoring.coreos.com_v1_prometheusrule.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| apiVersion: monitoring.coreos.com/v1 | ||
| kind: PrometheusRule | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/managed-by: operator-lifecycle-manager | ||
| app.kubernetes.io/name: opentelemetry-operator | ||
| app.kubernetes.io/part-of: opentelemetry-operator | ||
| name: opentelemetry-operator-prometheus-rules | ||
| spec: | ||
| groups: | ||
| - name: opentelemetry-operator-monitoring.rules | ||
| rules: | ||
| - expr: sum by (type) (opentelemetry_collector_receivers) | ||
| record: type:opentelemetry_collector_receivers:sum | ||
| - expr: sum by (type) (opentelemetry_collector_exporters) | ||
| record: type:opentelemetry_collector_exporters:sum | ||
| - expr: sum by (type) (opentelemetry_collector_processors) | ||
| record: type:opentelemetry_collector_processors:sum | ||
| - expr: sum by (type) (opentelemetry_collector_extensions) | ||
| record: type:opentelemetry_collector_extensions:sum | ||
| - expr: sum by (type) (opentelemetry_collector_connectors) | ||
| record: type:opentelemetry_collector_connectors:sum | ||
| - expr: sum by (type) (opentelemetry_collector_info) | ||
| record: type:opentelemetry_collector_info:sum |
15 changes: 15 additions & 0 deletions
15
...nshift/manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_role.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| kind: Role | ||
| metadata: | ||
| name: opentelemetry-operator-prometheus | ||
| rules: | ||
| - apiGroups: | ||
| - "" | ||
| resources: | ||
| - services | ||
| - endpoints | ||
| - pods | ||
| verbs: | ||
| - get | ||
| - list | ||
| - watch |
12 changes: 12 additions & 0 deletions
12
...manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_rolebinding.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| kind: RoleBinding | ||
| metadata: | ||
| name: opentelemetry-operator-prometheus | ||
| roleRef: | ||
| apiGroup: rbac.authorization.k8s.io | ||
| kind: Role | ||
| name: opentelemetry-operator-prometheus | ||
| subjects: | ||
| - kind: ServiceAccount | ||
| name: prometheus-k8s | ||
| namespace: openshift-monitoring |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
config/overlays/openshift/manager_auth_proxy_tls_patch.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: controller-manager | ||
| namespace: system | ||
| spec: | ||
| template: | ||
| spec: | ||
| containers: | ||
| - name: manager # without this line, kustomize reorders the containers, making kube-rbac-proxy the default container | ||
| - name: kube-rbac-proxy | ||
| args: | ||
| - "--secure-listen-address=0.0.0.0:8443" | ||
| - "--upstream=http://127.0.0.1:8080/" | ||
| - "--logtostderr=true" | ||
| - "--v=0" | ||
| - "--tls-cert-file=/var/run/tls/server/tls.crt" | ||
| - "--tls-private-key-file=/var/run/tls/server/tls.key" | ||
| - "--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA256" | ||
| - "--tls-min-version=VersionTLS12" | ||
| volumeMounts: | ||
| - mountPath: /var/run/tls/server | ||
| name: opentelemetry-operator-metrics-cert | ||
| volumes: | ||
| - name: opentelemetry-operator-metrics-cert | ||
| secret: | ||
| defaultMode: 420 | ||
| # secret generated by the 'service.beta.openshift.io/serving-cert-secret-name' annotation on the metrics-service | ||
| secretName: opentelemetry-operator-metrics |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| annotations: | ||
| service.beta.openshift.io/serving-cert-secret-name: opentelemetry-operator-metrics | ||
| name: controller-manager-metrics-service | ||
| namespace: system |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package operatormetrics | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/apimachinery/pkg/util/intstr" | ||
| "k8s.io/client-go/rest" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/manager" | ||
| ) | ||
|
|
||
| var ( | ||
| // namespaceFile is the path to the namespace file for the service account. | ||
| namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" | ||
|
|
||
| // caBundleConfigMap declares the name of the config map for the CA bundle. | ||
| caBundleConfigMap = "serving-certs-ca-bundle" | ||
|
|
||
| // prometheusCAFile declares the path for prometheus CA file for service monitors in OpenShift. | ||
| prometheusCAFile = fmt.Sprintf("/etc/prometheus/configmaps/%s/service-ca.crt", caBundleConfigMap) | ||
|
|
||
| // nolint #nosec | ||
| // bearerTokenFile declares the path for bearer token file for service monitors. | ||
| bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" | ||
|
|
||
| // openshiftInClusterMonitoringNamespace declares the namespace for the OpenShift in-cluster monitoring. | ||
| openshiftInClusterMonitoringNamespace = "openshift-monitoring" | ||
| ) | ||
|
|
||
| var _ manager.Runnable = &OperatorMetrics{} | ||
|
|
||
| type OperatorMetrics struct { | ||
| kubeClient client.Client | ||
| } | ||
|
|
||
| func NewOperatorMetrics(config *rest.Config, scheme *runtime.Scheme) (OperatorMetrics, error) { | ||
| kubeClient, err := client.New(config, client.Options{Scheme: scheme}) | ||
| if err != nil { | ||
| return OperatorMetrics{}, err | ||
| } | ||
|
|
||
| return OperatorMetrics{ | ||
| kubeClient: kubeClient, | ||
| }, nil | ||
| } | ||
|
|
||
| func (om OperatorMetrics) Start(ctx context.Context) error { | ||
| rawNamespace, err := os.ReadFile(namespaceFile) | ||
| if err != nil { | ||
| return fmt.Errorf("error reading namespace file: %w", err) | ||
| } | ||
| namespace := string(rawNamespace) | ||
|
|
||
| var tlsConfig *monitoringv1.TLSConfig | ||
|
|
||
| if om.caConfigMapExists() { | ||
| serviceName := fmt.Sprintf("opentelemetry-operator-controller-manager-metrics-service.%s.svc", namespace) | ||
|
|
||
| tlsConfig = &monitoringv1.TLSConfig{ | ||
| CAFile: prometheusCAFile, | ||
| SafeTLSConfig: monitoringv1.SafeTLSConfig{ | ||
| ServerName: &serviceName, | ||
| }, | ||
| } | ||
| } else { | ||
| t := true | ||
| tlsConfig = &monitoringv1.TLSConfig{ | ||
| SafeTLSConfig: monitoringv1.SafeTLSConfig{ | ||
| // kube-rbac-proxy uses a self-signed cert by default | ||
| InsecureSkipVerify: &t, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| sm := monitoringv1.ServiceMonitor{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "opentelemetry-operator-metrics-monitor", | ||
| Namespace: namespace, | ||
| Labels: map[string]string{ | ||
| "app.kubernetes.io/name": "opentelemetry-operator", | ||
| "app.kubernetes.io/part-of": "opentelemetry-operator", | ||
| "control-plane": "controller-manager", | ||
| }, | ||
| }, | ||
| Spec: monitoringv1.ServiceMonitorSpec{ | ||
| Selector: metav1.LabelSelector{ | ||
| MatchLabels: map[string]string{ | ||
| "app.kubernetes.io/name": "opentelemetry-operator", | ||
| }, | ||
| }, | ||
| Endpoints: []monitoringv1.Endpoint{ | ||
| { | ||
| BearerTokenFile: bearerTokenFile, | ||
| Interval: "30s", | ||
| Path: "/metrics", | ||
| Scheme: "https", | ||
| ScrapeTimeout: "10s", | ||
| TargetPort: &intstr.IntOrString{IntVal: 8443}, | ||
| TLSConfig: tlsConfig, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| err = om.kubeClient.Create(ctx, &sm) | ||
| if err != nil { | ||
| return fmt.Errorf("error creating service monitor: %w", err) | ||
| } | ||
|
|
||
| <-ctx.Done() | ||
|
|
||
| return om.kubeClient.Delete(ctx, &sm) | ||
| } | ||
|
|
||
| func (om OperatorMetrics) NeedLeaderElection() bool { | ||
| return true | ||
| } | ||
|
|
||
| func (om OperatorMetrics) caConfigMapExists() bool { | ||
| return om.kubeClient.Get(context.Background(), client.ObjectKey{ | ||
| Name: caBundleConfigMap, | ||
| Namespace: openshiftInClusterMonitoringNamespace, | ||
| }, &corev1.ConfigMap{}, | ||
| ) == nil | ||
| } |
Oops, something went wrong.