From 3096d2ea2f4718a5259e8e0facd327b5a7b1b5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20SANCHEZ?= Date: Mon, 21 Nov 2022 10:26:47 +0100 Subject: [PATCH 1/3] feat(helm): add autoscaling to helm chart --- charts/metrics-server/README.md | 121 ++++++++++-------- charts/metrics-server/templates/_helpers.tpl | 7 + .../templates/clusterrole-nanny.yaml | 13 ++ .../templates/clusterrolebinding-nanny.yaml | 18 +++ .../templates/configmaps-nanny.yaml | 17 +++ .../metrics-server/templates/deployment.yaml | 31 +++++ .../metrics-server/templates/role-nanny.yaml | 27 ++++ .../templates/rolebinding-nanny.yaml | 19 +++ charts/metrics-server/values.yaml | 21 +++ 9 files changed, 219 insertions(+), 55 deletions(-) create mode 100644 charts/metrics-server/templates/clusterrole-nanny.yaml create mode 100644 charts/metrics-server/templates/clusterrolebinding-nanny.yaml create mode 100644 charts/metrics-server/templates/configmaps-nanny.yaml create mode 100644 charts/metrics-server/templates/role-nanny.yaml create mode 100644 charts/metrics-server/templates/rolebinding-nanny.yaml diff --git a/charts/metrics-server/README.md b/charts/metrics-server/README.md index 7b2334138..a10cbae5f 100644 --- a/charts/metrics-server/README.md +++ b/charts/metrics-server/README.md @@ -22,58 +22,69 @@ helm upgrade --install metrics-server metrics-server/metrics-server The following table lists the configurable parameters of the _Metrics Server_ chart and their default values. -| Parameter | Description | Default | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | -| `image.repository` | Image repository. | `registry.k8s.io/metrics-server/metrics-server` | -| `image.tag` | Image tag, will override the default tag derived from the chart app version. | `""` | -| `image.pullPolicy` | Image pull policy. | `IfNotPresent` | -| `imagePullSecrets` | Image pull secrets. | `[]` | -| `nameOverride` | Override the `name` of the chart. | `nil` | -| `fullnameOverride` | Override the `fullname` of the chart. | `nil` | -| `serviceAccount.create` | If `true`, create a new service account. | `true` | -| `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | -| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template. | `nil` | -| `serviceAccount.secrets` | The list of secrets mountable by this service account. See https://kubernetes.io/docs/reference/labels-annotations-taints/#enforce-mountable-secrets | `[]` | -| `rbac.create` | If `true`, create the RBAC resources. | `true` | -| `rbac.pspEnabled` | If `true`, create a pod security policy resource. | `false` | -| `apiService.create` | If `true`, create the `v1beta1.metrics.k8s.io` API service. You typically want this enabled! If you disable API service creation you have to manage it outside of this chart for e.g horizontal pod autoscaling to work with this release. | `true` | -| `apiService.annotations` | Annotations to add to the API service. | `{}` | -| `apiService.insecureSkipTLSVerify` | Specifies whether to skip TLS verification | `true` | -| `apiService.caBundle` | The PEM encoded CA bundle for TLS verification | `""` | -| `commonLabels` | Labels to add to each object of the chart. | `{}` | -| `podLabels` | Labels to add to the pod. | `{}` | -| `podAnnotations` | Annotations to add to the pod. | `{}` | -| `podSecurityContext` | Security context for the pod. | `{}` | -| `securityContext` | Security context for the _metrics-server_ container. | _See values.yaml_ | -| `priorityClassName` | Priority class name to use. | `system-cluster-critical` | -| `containerPort` | port for the _metrics-server_ container. | `4443` | -| `hostNetwork.enabled` | If `true`, start _metric-server_ in hostNetwork mode. You would require this enabled if you use alternate overlay networking for pods and API server unable to communicate with metrics-server. As an example, this is required if you use Weave network on EKS. | `false` | -| `replicas` | Number of replicas to run. | `1` | -| `updateStrategy` | Customise the default update strategy. | `{}` | -| `podDisruptionBudget.enabled` | If `true`, create `PodDisruptionBudget` resource. | `{}` | -| `podDisruptionBudget.minAvailable` | Set the `PodDisruptionBugdet` minimum available pods. | `nil` | -| `podDisruptionBudget.maxUnavailable` | Set the `PodDisruptionBugdet` maximum unavailable pods. | `nil` | -| `defaultArgs` | Default arguments to pass to the _metrics-server_ command. | See _values.yaml_ | -| `args` | Additional arguments to pass to the _metrics-server_ command. | `[]` | -| `livenessProbe` | Liveness probe. | See _values.yaml_ | -| `readinessProbe` | Readiness probe. | See _values.yaml_ | -| `service.type` | Service type. | `ClusterIP` | -| `service.port` | Service port. | `443` | -| `service.annotations` | Annotations to add to the service. | `{}` | -| `service.labels` | Labels to add to the service. | `{}` | -| `metrics.enabled` | If `true`, allow unauthenticated access to `/metrics`. | `false` | -| `serviceMonitor.enabled` | If `true`, create a _Prometheus_ service monitor. This needs `metrics.enabled` to be `true`. | `false` | -| `serviceMonitor.additionalLabels` | Additional labels to be set on the ServiceMonitor. | `{}` | -| `serviceMonitor.metricRelabelings` | _Prometheus_ metric relabeling. | `[]` | -| `serviceMonitor.relabelings` | _Prometheus_ relabeling. | `[]` | -| `serviceMonitor.interval` | _Prometheus_ scrape frequency. | `1m` | -| `serviceMonitor.scrapeTimeout` | _Prometheus_ scrape timeout. | `10s` | -| `resources` | Resource requests and limits for the _metrics-server_ container. See https://github.com/kubernetes-sigs/metrics-server#scaling | `{}` | -| `extraVolumeMounts` | Additional volume mounts for the _metrics-server_ container. | `[]` | -| `extraVolumes` | Additional volumes for the pod. | `[]` | -| `nodeSelector` | Node labels for pod assignment. | `{}` | -| `tolerations` | Tolerations for pod assignment. | `[]` | -| `affinity` | Affinity for pod assignment. | `{}` | -| `topologySpreadConstraints` | Pod Topology Spread Constraints. | `[]` | -| `deploymentAnnotations` | Annotations to add to the deployment. | `{}` | -| `schedulerName` | scheduler to set to the deployment. | `""` | +| Parameter | Description | Default | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| `image.repository` | Image repository. | `registry.k8s.io/metrics-server/metrics-server` | +| `image.tag` | Image tag, will override the default tag derived from the chart app version. | `""` | +| `image.pullPolicy` | Image pull policy. | `IfNotPresent` | +| `imagePullSecrets` | Image pull secrets. | `[]` | +| `nameOverride` | Override the `name` of the chart. | `nil` | +| `fullnameOverride` | Override the `fullname` of the chart. | `nil` | +| `serviceAccount.create` | If `true`, create a new service account. | `true` | +| `serviceAccount.annotations` | Annotations to add to the service account. | `{}` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template. | `nil` | +| `serviceAccount.secrets` | The list of secrets mountable by this service account. See https://kubernetes.io/docs/reference/labels-annotations-taints/#enforce-mountable-secrets | `[]` | +| `rbac.create` | If `true`, create the RBAC resources. | `true` | +| `rbac.pspEnabled` | If `true`, create a pod security policy resource. | `false` | +| `apiService.create` | If `true`, create the `v1beta1.metrics.k8s.io` API service. You typically want this enabled! If you disable API service creation you have to manage it outside of this chart for e.g horizontal pod autoscaling to work with this release. | `true` | +| `apiService.annotations` | Annotations to add to the API service | `{}` | +| `apiService.insecureSkipTLSVerify` | Specifies whether to skip TLS verification | `true` | +| `apiService.caBundle` | The PEM encoded CA bundle for TLS verification | `""` | +| `commonLabels` | Labels to add to each object of the chart. | `{}` | +| `podLabels` | Labels to add to the pod. | `{}` | +| `podAnnotations` | Annotations to add to the pod. | `{}` | +| `podSecurityContext` | Security context for the pod. | `{}` | +| `securityContext` | Security context for the _metrics-server_ container. | _See values.yaml_ | +| `priorityClassName` | Priority class name to use. | `system-cluster-critical` | +| `containerPort` | port for the _metrics-server_ container. | `10250` | +| `hostNetwork.enabled` | If `true`, start _metric-server_ in hostNetwork mode. You would require this enabled if you use alternate overlay networking for pods and API server unable to communicate with metrics-server. As an example, this is required if you use Weave network on EKS. | `false` | +| `replicas` | Number of replicas to run. | `1` | +| `updateStrategy` | Customise the default update strategy. | `{}` | +| `podDisruptionBudget.enabled` | If `true`, create `PodDisruptionBudget` resource. | `{}` | +| `podDisruptionBudget.minAvailable` | Set the `PodDisruptionBugdet` minimum available pods. | `nil` | +| `podDisruptionBudget.maxUnavailable` | Set the `PodDisruptionBugdet` maximum unavailable pods. | `nil` | +| `defaultArgs` | Default arguments to pass to the _metrics-server_ command. | See _values.yaml_ | +| `args` | Additional arguments to pass to the _metrics-server_ command. | `[]` | +| `livenessProbe` | Liveness probe. | See _values.yaml_ | +| `readinessProbe` | Readiness probe. | See _values.yaml_ | +| `service.type` | Service type. | `ClusterIP` | +| `service.port` | Service port. | `443` | +| `service.annotations` | Annotations to add to the service. | `{}` | +| `service.labels` | Labels to add to the service. | `{}` | +| `addonResizer.enabled` | If `true`, run the addon-resizer as a sidecar to automatically scale resource requests with cluster size. | `false` | +| `addonResizer.image.repository` | addon-resizer image repository | registry.k8s.io/autoscaling/addon-resizer | +| `addonResizer.image.tag` | addon-resizer image tag | 1.8.14 | +| `addonResizer.resources` | Resource requests and limits for the _nanny_ container. | `{limits: {cpu: 40m, memory: 25Mi}, requests: {cpu: 40m, memory: 25Mi}}` | +| `addonResizer.nanny.cpu` | The base CPU requirement. | 20m | +| `addonResizer.nanny.extraCPU` | The amount of CPU to add per node. | 1m | +| `addonResizer.nanny.extraMemory` | The amount of memory to add per node. | 2Mi | +| `addonResizer.nanny.memory` | The base memory requirement. | 15Mi | +| `addonResizer.nanny.minClusterSize` | Specifies the smallest number of nodes resources will be scaled to. | 10 | +| `addonResizer.nanny.pollPeriod` | The time, in milliseconds, to poll the dependent container. | 300000 | +| `addonResizer.nanny.threshold` | A number between 0-100. The dependent's resources are rewritten when they deviate from expected by more than threshold. | 5 | +| `metrics.enabled` | If `true`, allow unauthenticated access to `/metrics`. | `false` | +| `serviceMonitor.enabled` | If `true`, create a _Prometheus_ service monitor. This needs `metrics.enabled` to be `true`. | `false` | +| `serviceMonitor.additionalLabels` | Additional labels to be set on the ServiceMonitor. | `{}` | +| `serviceMonitor.metricRelabelings` | _Prometheus_ metric relabeling. | `[]` | +| `serviceMonitor.relabelings` | _Prometheus_ relabeling. | `[]` | +| `serviceMonitor.interval` | _Prometheus_ scrape frequency. | `1m` | +| `serviceMonitor.scrapeTimeout` | _Prometheus_ scrape timeout. | `10s` | +| `resources` | Resource requests and limits for the _metrics-server_ container. See https://github.com/kubernetes-sigs/metrics-server#scaling | `{}` | +| `extraVolumeMounts` | Additional volume mounts for the _metrics-server_ container. | `[]` | +| `extraVolumes` | Additional volumes for the pod. | `[]` | +| `nodeSelector` | Node labels for pod assignment. | `{}` | +| `tolerations` | Tolerations for pod assignment. | `[]` | +| `affinity` | Affinity for pod assignment. | `{}` | +| `topologySpreadConstraints` | Pod Topology Spread Constraints. | `[]` | +| `deploymentAnnotations` | Annotations to add to the deployment. | `{}` | +| `schedulerName` | scheduler to set to the deployment. | `""` | diff --git a/charts/metrics-server/templates/_helpers.tpl b/charts/metrics-server/templates/_helpers.tpl index b154ab5d3..2c8a47ca6 100644 --- a/charts/metrics-server/templates/_helpers.tpl +++ b/charts/metrics-server/templates/_helpers.tpl @@ -71,6 +71,13 @@ The image to use {{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} {{- end }} +{{/* +The image to use for the addon resizer +*/}} +{{- define "metrics-server.addonResizer.image" -}} +{{- printf "%s:%s" .Values.addonResizer.image.repository .Values.addonResizer.image.tag }} +{{- end }} + {{/* Get PodDisruptionBudget API Version */}} {{- define "metrics-server.pdb.apiVersion" -}} {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} diff --git a/charts/metrics-server/templates/clusterrole-nanny.yaml b/charts/metrics-server/templates/clusterrole-nanny.yaml new file mode 100644 index 000000000..24edd81c5 --- /dev/null +++ b/charts/metrics-server/templates/clusterrole-nanny.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.rbac.create .Values.addonResizer.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ printf "system:%s-nanny" (include "metrics-server.fullname" .) }} + labels: + {{- include "metrics-server.labels" . | nindent 4 }} +rules: + - nonResourceURLs: + - /metrics + verbs: + - get +{{- end -}} diff --git a/charts/metrics-server/templates/clusterrolebinding-nanny.yaml b/charts/metrics-server/templates/clusterrolebinding-nanny.yaml new file mode 100644 index 000000000..43738ccbf --- /dev/null +++ b/charts/metrics-server/templates/clusterrolebinding-nanny.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.addonResizer.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ printf "system:%s-nanny" (include "metrics-server.fullname" .) }} + labels: + {{- include "metrics-server.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:{{ template "metrics-server.fullname" . }}-nanny +subjects: + - kind: ServiceAccount + name: {{ include "metrics-server.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} +{{- end -}} diff --git a/charts/metrics-server/templates/configmaps-nanny.yaml b/charts/metrics-server/templates/configmaps-nanny.yaml new file mode 100644 index 000000000..10164820a --- /dev/null +++ b/charts/metrics-server/templates/configmaps-nanny.yaml @@ -0,0 +1,17 @@ +{{- if .Values.addonResizer.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "metrics-server.fullname" .}}-nanny-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "metrics-server.labels" . | nindent 4 }} +data: + NannyConfiguration: |- + apiVersion: nannyconfig/v1alpha1 + kind: NannyConfiguration + baseCPU: {{ .Values.addonResizer.nanny.cpu }} + cpuPerNode: {{ .Values.addonResizer.nanny.extraCpu }} + baseMemory: {{ .Values.addonResizer.nanny.memory }} + memoryPerNode: {{ .Values.addonResizer.nanny.extraMemory }} +{{- end -}} diff --git a/charts/metrics-server/templates/deployment.yaml b/charts/metrics-server/templates/deployment.yaml index 8d099749d..2bd591784 100644 --- a/charts/metrics-server/templates/deployment.yaml +++ b/charts/metrics-server/templates/deployment.yaml @@ -87,6 +87,37 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.addonResizer.enabled }} + - name: metrics-server-nanny + image: {{ include "metrics-server.addonResizer.image" . }} + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /pod_nanny + - --config-dir=/etc/config + - --deployment={{ include "metrics-server.fullname" . }} + - --threshold={{ .Values.addonResizer.nanny.threshold }} + - --deployment={{ include "metrics-server.fullname" . }} + - --container=metrics-server + - --poll-period={{ .Values.addonResizer.nanny.pollPeriod }} + - --estimator=exponential + - --minClusterSize={{ .Values.addonResizer.nanny.minClusterSize }} + - --use-metrics=true + volumeMounts: + - name: nanny-config-volume + mountPath: /etc/config + {{- with .Values.addonResizer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} volumes: - name: tmp emptyDir: {} diff --git a/charts/metrics-server/templates/role-nanny.yaml b/charts/metrics-server/templates/role-nanny.yaml new file mode 100644 index 000000000..8551b3713 --- /dev/null +++ b/charts/metrics-server/templates/role-nanny.yaml @@ -0,0 +1,27 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.addonResizer.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "system:%s-nanny" (include "metrics-server.fullname" .) }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "metrics-server.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - metrics-server + verbs: + - get + - patch +{{- end -}} +{{- end -}} diff --git a/charts/metrics-server/templates/rolebinding-nanny.yaml b/charts/metrics-server/templates/rolebinding-nanny.yaml new file mode 100644 index 000000000..99570ac62 --- /dev/null +++ b/charts/metrics-server/templates/rolebinding-nanny.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.addonResizer.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-nanny" (include "metrics-server.fullname" .) }} + namespace: kube-system + labels: + {{- include "metrics-server.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: metrics-server-nanny +subjects: + - kind: ServiceAccount + name: {{ include "metrics-server.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} +{{- end -}} diff --git a/charts/metrics-server/values.yaml b/charts/metrics-server/values.yaml index fd7d48011..9741208b8 100644 --- a/charts/metrics-server/values.yaml +++ b/charts/metrics-server/values.yaml @@ -118,6 +118,27 @@ service: # kubernetes.io/cluster-service: "true" # kubernetes.io/name: "Metrics-server" +addonResizer: + enabled: false + image: + repository: registry.k8s.io/autoscaling/addon-resizer + tag: 1.8.14 + resources: + limits: + cpu: 40m + memory: 25Mi + requests: + cpu: 40m + memory: 25Mi + nanny: + cpu: 20m + extraCpu: 1m + extraMemory: 2Mi + memory: 15Mi + minClusterSize: 10 + pollPeriod: 300000 + threshold: 5 + metrics: enabled: false From 9e29bbd0406ee787caa52524aac3287b15d8788b Mon Sep 17 00:00:00 2001 From: JoaoBraveCoding Date: Thu, 23 Mar 2023 09:51:55 +0000 Subject: [PATCH 2/3] helm: fixes service annotation Issue: https://github.com/kubernetes-sigs/metrics-server/issues/1219 Problem: when labels for the service are defined they will also show in the annotations Signed-off-by: JoaoBraveCoding --- charts/metrics-server/templates/service.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/metrics-server/templates/service.yaml b/charts/metrics-server/templates/service.yaml index 35d3373d8..d45bcf36a 100644 --- a/charts/metrics-server/templates/service.yaml +++ b/charts/metrics-server/templates/service.yaml @@ -5,11 +5,11 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "metrics-server.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} - annotations: + {{- with .Values.service.labels -}} {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.service.labels -}} + {{- with .Values.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: From d32d0c96ae9175e207274a77397a5cd1a5fbe6a3 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Mon, 27 Mar 2023 19:04:10 +0100 Subject: [PATCH 3/3] chore(chart): Defined chart release v3.9.0 Signed-off-by: Steve Hipwell --- charts/metrics-server/Chart.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/charts/metrics-server/Chart.yaml b/charts/metrics-server/Chart.yaml index 160811956..3fe95c010 100644 --- a/charts/metrics-server/Chart.yaml +++ b/charts/metrics-server/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: metrics-server description: Metrics Server is a scalable, efficient source of container resource metrics for Kubernetes built-in autoscaling pipelines. type: application -version: 3.8.4 -appVersion: 0.6.2 +version: 3.9.0 +appVersion: 0.6.3 keywords: - kubernetes - metrics-server @@ -21,5 +21,9 @@ maintainers: url: https://github.com/endrec annotations: artifacthub.io/changes: | + - kind: added + description: "Added autoscaling support via the addon-resizer." - kind: changed - description: "Changed the image registry location to registry.k8s.io." + description: "Updated the Metrics Server OCI image to v0.6.3." + - kind: fixed + description: "Fixed service labels/annotations."