From 6781705de65a3a71482a67da51f709f8a8eb7c34 Mon Sep 17 00:00:00 2001 From: Pavel Busko Date: Wed, 16 Oct 2024 11:59:59 +0200 Subject: [PATCH] extract statefulset-runner to a standalone deployment --- controllers/config/config.go | 3 - controllers/main.go | 23 ---- helm/korifi/controllers/configmap.yaml | 1 - helm/korifi/controllers/rbac.yaml | 16 --- .../korifi/statefulset-runner/deployment.yaml | 68 ++++++++++ .../post-install-runnerinfo.yaml | 2 +- helm/korifi/statefulset-runner/rbac.yaml | 36 ++++++ helm/korifi/statefulset-runner/service.yaml | 18 +++ helm/korifi/values.yaml | 9 +- scripts/assets/korifi-debug-kbld.yml | 6 + scripts/assets/korifi-kbld.yml | 6 + statefulset-runner/Dockerfile | 32 +++++ statefulset-runner/main.go | 121 ++++++++++++++++++ statefulset-runner/package.go | 7 - statefulset-runner/remote-debug/Dockerfile | 35 +++++ tests/crds/apps_test.go | 4 +- 16 files changed, 330 insertions(+), 57 deletions(-) create mode 100644 helm/korifi/statefulset-runner/deployment.yaml create mode 100644 helm/korifi/statefulset-runner/rbac.yaml create mode 100644 helm/korifi/statefulset-runner/service.yaml create mode 100644 statefulset-runner/Dockerfile create mode 100644 statefulset-runner/main.go delete mode 100644 statefulset-runner/package.go create mode 100644 statefulset-runner/remote-debug/Dockerfile diff --git a/controllers/config/config.go b/controllers/config/config.go index 53b9115ec..f932454af 100644 --- a/controllers/config/config.go +++ b/controllers/config/config.go @@ -9,9 +9,6 @@ import ( ) type ControllerConfig struct { - // components - IncludeStatefulsetRunner bool `yaml:"includeStatefulsetRunner"` - // core controllers CFProcessDefaults CFProcessDefaults `yaml:"cfProcessDefaults"` CFStagingResources CFStagingResources `yaml:"cfStagingResources"` diff --git a/controllers/main.go b/controllers/main.go index 7bd7a957e..aa798b150 100644 --- a/controllers/main.go +++ b/controllers/main.go @@ -60,7 +60,6 @@ import ( packageswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/packages" spaceswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/spaces" taskswebhook "code.cloudfoundry.org/korifi/controllers/webhooks/workloads/tasks" - statefulsetcontrollers "code.cloudfoundry.org/korifi/statefulset-runner/controllers" "code.cloudfoundry.org/korifi/tools" "code.cloudfoundry.org/korifi/tools/image" "code.cloudfoundry.org/korifi/version" @@ -333,28 +332,6 @@ func main() { os.Exit(1) } - if controllerConfig.IncludeStatefulsetRunner { - if err = statefulsetcontrollers.NewAppWorkloadReconciler( - mgr.GetClient(), - mgr.GetScheme(), - statefulsetcontrollers.NewAppWorkloadToStatefulsetConverter(mgr.GetScheme()), - statefulsetcontrollers.NewPDBUpdater(mgr.GetClient()), - controllersLog, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AppWorkload") - os.Exit(1) - } - - if err = statefulsetcontrollers.NewRunnerInfoReconciler( - mgr.GetClient(), - mgr.GetScheme(), - controllersLog, - ).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "RunnerInfo") - os.Exit(1) - } - } - if err = routes.NewReconciler( mgr.GetClient(), mgr.GetScheme(), diff --git a/helm/korifi/controllers/configmap.yaml b/helm/korifi/controllers/configmap.yaml index c4ea0d61a..6400d8677 100644 --- a/helm/korifi/controllers/configmap.yaml +++ b/helm/korifi/controllers/configmap.yaml @@ -5,7 +5,6 @@ metadata: namespace: {{ .Release.Namespace }} data: config.yaml: |- - includeStatefulsetRunner: {{ .Values.statefulsetRunner.include }} builderName: {{ .Values.reconcilers.build }} runnerName: {{ .Values.reconcilers.run }} cfProcessDefaults: diff --git a/helm/korifi/controllers/rbac.yaml b/helm/korifi/controllers/rbac.yaml index 8071736ed..75bf0a4f1 100644 --- a/helm/korifi/controllers/rbac.yaml +++ b/helm/korifi/controllers/rbac.yaml @@ -79,19 +79,3 @@ subjects: - kind: ServiceAccount name: korifi-controllers-controller-manager namespace: {{ .Release.Namespace }} - -{{- if .Values.statefulsetRunner.include }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: korifi-statefulset-runner-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: korifi-statefulset-runner-appworkload-manager-role -subjects: -- kind: ServiceAccount - name: korifi-controllers-controller-manager - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/helm/korifi/statefulset-runner/deployment.yaml b/helm/korifi/statefulset-runner/deployment.yaml new file mode 100644 index 000000000..665ae98c4 --- /dev/null +++ b/helm/korifi/statefulset-runner/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: statefulset-runner + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.statefulsetRunner.replicas }} + selector: + matchLabels: + app: statefulset-runner + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + prometheus.io/path: /metrics + prometheus.io/port: "8080" + prometheus.io/scrape: "true" + labels: + app: statefulset-runner + spec: + containers: + - name: manager + image: {{ .Values.statefulsetRunner.image }} +{{- if .Values.debug }} + command: + - "/dlv" + args: + - "--listen=:40000" + - "--headless=true" + - "--api-version=2" + - "exec" + - "/manager" + - "--continue" + - "--accept-multiclient" + - "--" + - "--health-probe-bind-address=:8081" + - "--leader-elect" +{{- else }} + args: + - --health-probe-bind-address=:8081 + - --leader-elect +{{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + ports: + - containerPort: 8080 + name: metrics + protocol: TCP + resources: + {{- .Values.statefulsetRunner.resources | toYaml | nindent 10 }} + {{- include "korifi.securityContext" . | indent 8 }} + {{- include "korifi.podSecurityContext" . | indent 6 }} + serviceAccountName: statefulset-runner-controller-manager +{{- if .Values.statefulsetRunner.nodeSelector }} + nodeSelector: + {{ toYaml .Values.statefulsetRunner.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.statefulsetRunner.tolerations }} + tolerations: + {{- toYaml .Values.statefulsetRunner.tolerations | nindent 8 }} +{{- end }} + terminationGracePeriodSeconds: 10 diff --git a/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml b/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml index 1179f55e3..fb845ba64 100644 --- a/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml +++ b/helm/korifi/statefulset-runner/post-install-runnerinfo.yaml @@ -23,7 +23,7 @@ spec: app.kubernetes.io/instance: {{ .Release.Name | quote }} helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" spec: - serviceAccountName: korifi-controllers-controller-manager + serviceAccountName: statefulset-runner-controller-manager restartPolicy: Never {{- include "korifi.podSecurityContext" . | indent 6 }} containers: diff --git a/helm/korifi/statefulset-runner/rbac.yaml b/helm/korifi/statefulset-runner/rbac.yaml new file mode 100644 index 000000000..e5ebde3ec --- /dev/null +++ b/helm/korifi/statefulset-runner/rbac.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +imagePullSecrets: +{{- range .Values.systemImagePullSecrets }} +- name: {{ . | quote }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: statefulset-runner-leader-election-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: korifi-controllers-leader-election-role +subjects: +- kind: ServiceAccount + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: statefulset-runner-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: korifi-statefulset-runner-appworkload-manager-role +subjects: +- kind: ServiceAccount + name: statefulset-runner-controller-manager + namespace: {{ .Release.Namespace }} diff --git a/helm/korifi/statefulset-runner/service.yaml b/helm/korifi/statefulset-runner/service.yaml new file mode 100644 index 000000000..b8e449281 --- /dev/null +++ b/helm/korifi/statefulset-runner/service.yaml @@ -0,0 +1,18 @@ +--- +{{- if .Values.debug }} +apiVersion: v1 +kind: Service +metadata: + name: statefulset-runner-debug-port + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: debug-30055 + nodePort: 30055 + port: 30055 + protocol: TCP + targetPort: 40000 + selector: + app: statefulset-runner + type: NodePort +{{- end }} diff --git a/helm/korifi/values.yaml b/helm/korifi/values.yaml index a4f60ae97..cf5cb5591 100644 --- a/helm/korifi/values.yaml +++ b/helm/korifi/values.yaml @@ -112,14 +112,15 @@ kpackImageBuilder: statefulsetRunner: include: true + image: cloudfoundry/statefulset-runner:latest replicas: 1 resources: limits: - cpu: 500m - memory: 128Mi + cpu: 1000m + memory: 1Gi requests: - cpu: 10m - memory: 64Mi + cpu: 50m + memory: 100Mi jobTaskRunner: include: true diff --git a/scripts/assets/korifi-debug-kbld.yml b/scripts/assets/korifi-debug-kbld.yml index cf19ef1d8..f017fc796 100644 --- a/scripts/assets/korifi-debug-kbld.yml +++ b/scripts/assets/korifi-debug-kbld.yml @@ -20,6 +20,12 @@ sources: buildx: file: kpack-image-builder/remote-debug/Dockerfile +- image: cloudfoundry/statefulset-runner:latest + path: . + docker: + buildx: + file: statefulset-runner/remote-debug/Dockerfile + - image: cloudfoundry/job-task-runner:latest path: . docker: diff --git a/scripts/assets/korifi-kbld.yml b/scripts/assets/korifi-kbld.yml index 359f62d3f..7a54c0cc5 100644 --- a/scripts/assets/korifi-kbld.yml +++ b/scripts/assets/korifi-kbld.yml @@ -20,6 +20,12 @@ sources: buildx: file: kpack-image-builder/Dockerfile +- image: cloudfoundry/statefulset-runner:latest + path: . + docker: + buildx: + file: statefulset-runner/Dockerfile + - image: cloudfoundry/job-task-runner:latest path: . docker: diff --git a/statefulset-runner/Dockerfile b/statefulset-runner/Dockerfile new file mode 100644 index 000000000..e4b383d49 --- /dev/null +++ b/statefulset-runner/Dockerfile @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY statefulset-runner statefulset-runner +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -o manager statefulset-runner/main.go + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot + +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/statefulset-runner/main.go b/statefulset-runner/main.go new file mode 100644 index 000000000..2e6a4a904 --- /dev/null +++ b/statefulset-runner/main.go @@ -0,0 +1,121 @@ +package main + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate -o fake -fake-name Client sigs.k8s.io/controller-runtime/pkg/client.Client +//counterfeiter:generate -o fake -fake-name EventRecorder k8s.io/client-go/tools/record.EventRecorder +//counterfeiter:generate -o fake -fake-name StatusWriter sigs.k8s.io/controller-runtime/pkg/client.StatusWriter + +import ( + "flag" + "fmt" + "os" + + korifiv1alpha1 "code.cloudfoundry.org/korifi/controllers/api/v1alpha1" + statefulsetcontrollers "code.cloudfoundry.org/korifi/statefulset-runner/controllers" + "code.cloudfoundry.org/korifi/tools" + "go.uber.org/zap/zapcore" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(korifiv1alpha1.AddToScheme(scheme)) +} + +func main() { + var ( + metricsAddr string + enableLeaderElection bool + probeAddr string + ) + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.Parse() + + logger, _, err := tools.NewZapLogger(zapcore.InfoLevel) + if err != nil { + panic(fmt.Sprintf("error creating new zap logger: %v", err)) + } + + ctrl.SetLogger(logger) + klog.SetLogger(ctrl.Log) + + conf := ctrl.GetConfigOrDie() + mgr, err := ctrl.NewManager(conf, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: 9443, + }), + Metrics: metricsserver.Options{ + BindAddress: metricsAddr, + }, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "13c200bs.cloudfoundry.org", + }) + if err != nil { + setupLog.Error(err, "unable to initialize manager") + os.Exit(1) + } + + if err := setupControllers(mgr); err != nil { + setupLog.Error(err, "unable to set up controllers") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +func setupControllers(mgr manager.Manager) error { + controllersLog := ctrl.Log.WithName("controllers") + if err := statefulsetcontrollers.NewAppWorkloadReconciler( + mgr.GetClient(), + mgr.GetScheme(), + statefulsetcontrollers.NewAppWorkloadToStatefulsetConverter( + mgr.GetScheme(), + false, + ), + statefulsetcontrollers.NewPDBUpdater(mgr.GetClient()), + controllersLog, + ).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create AppWorkload controller: %w", err) + } + + if err := statefulsetcontrollers.NewRunnerInfoReconciler( + mgr.GetClient(), + mgr.GetScheme(), + controllersLog, + ).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create RunnerInfo controller: %w", err) + } + + return nil +} diff --git a/statefulset-runner/package.go b/statefulset-runner/package.go deleted file mode 100644 index 5c472cba9..000000000 --- a/statefulset-runner/package.go +++ /dev/null @@ -1,7 +0,0 @@ -package statefulsetrunner - -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate - -//counterfeiter:generate -o fake -fake-name Client sigs.k8s.io/controller-runtime/pkg/client.Client -//counterfeiter:generate -o fake -fake-name EventRecorder k8s.io/client-go/tools/record.EventRecorder -//counterfeiter:generate -o fake -fake-name StatusWriter sigs.k8s.io/controller-runtime/pkg/client.StatusWriter diff --git a/statefulset-runner/remote-debug/Dockerfile b/statefulset-runner/remote-debug/Dockerfile new file mode 100644 index 000000000..97d6f6680 --- /dev/null +++ b/statefulset-runner/remote-debug/Dockerfile @@ -0,0 +1,35 @@ +# syntax = docker/dockerfile:experimental +FROM golang:1.23 as builder + +ARG version=dev + +WORKDIR /workspace + +COPY go.mod go.sum ./ + +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY api api +COPY controllers controllers +COPY statefulset-runner statefulset-runner +COPY model model +COPY tools tools +COPY version version + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux go build -ldflags "-X code.cloudfoundry.org/korifi/version.Version=${version}" -gcflags=all="-N -l" -o manager statefulset-runner/main.go + +# Get Delve from a GOPATH not from a Go Modules project +WORKDIR /go/src/ +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +FROM ubuntu + +WORKDIR / +COPY --from=builder /workspace/manager . +COPY --from=builder /go/bin/dlv . +EXPOSE 8080 8081 9443 40000 + +CMD ["/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/manager", "--continue", "--accept-multiclient"] diff --git a/tests/crds/apps_test.go b/tests/crds/apps_test.go index ef3935406..44b7852ec 100644 --- a/tests/crds/apps_test.go +++ b/tests/crds/apps_test.go @@ -185,9 +185,9 @@ func uploadAppBits(appGUID, packageGUID string) { controllersConfig := config.ControllerConfig{} Expect(yaml.Unmarshal([]byte(controllersConfigMap.Data["config.yaml"]), &controllersConfig)).To(Succeed()) - repoCreator := registry.NewRepositoryCreator(controllersConfig.ContainerRegistryType) + repoCreator := registry.NewRepositoryCreator("") Expect(repoCreator.CreateRepository(ctx, fmt.Sprintf("%s%s-packages", - controllersConfig.ContainerRepositoryPrefix, + "", appGUID, ))).To(Succeed())