Manage Pipelines Declaratively (#7)

.gitignore

@@ -0,0 +1,15 @@
+# Go
+bin/
+testbin/
+*.out
+/vendor/
+
+# VS Code
+/.vscode/
+
+# Python
+__pycache__/
+.python-version
+
+# Tests
+pids

Dockerfile

@@ -0,0 +1,27 @@
+# Build the manager binary
+FROM golang:1.16 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY apis/ apis/
+COPY controllers/ controllers/
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]

Makefile

@@ -0,0 +1,135 @@
+
+# Image URL to use all building/pushing image targets
+IMG ?= controller:latest
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
+CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Development
+
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+vet: ## Run go vet against code.
+	go vet ./...
+
+ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+decoupled-test: ## Run decoupled acceptance tests
+	mkdir -p ${ENVTEST_ASSETS_DIR}
+	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
+	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -tags=decoupled -coverprofile cover.out
+
+integration-test-up:
+	minikube start -p argo-integration-tests
+	kubectl create namespace argo --dry-run=client -o yaml | kubectl apply -f -
+	kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml
+	kubectl apply -n argo -f config/testing/wiremock.yaml
+	rm -f config/testing/pids
+	kubectl wait -n argo deployment/wiremock --for condition=available --timeout=5m
+	kubectl port-forward -n argo service/wiremock 8081:80 & echo $$! >> config/testing/pids
+	kubectl wait -n argo deployment/workflow-controller --for condition=available --timeout=5m
+	kubectl proxy --port=8080 & echo $$! >> config/testing/pids
+
+integration-test: ## Run integration tests
+	eval $$(minikube -p argo-integration-tests docker-env) && \
+	docker build compiler -t compiler && \
+	docker build kfp-tools -t kfp-tools && \
+	docker build docs/quickstart -t kfp-quickstart
+	go test ./... -tags=integration
+
+integration-test-down:
+	(cat config/testing/pids | xargs kill) || true
+	minikube stop -p argo-integration-tests
+
+unit-test: ## Run unit tests
+	go test ./... -tags=unit
+
+test: manifests generate fmt vet unit-test decoupled-test
+
+##@ Build
+
+build: generate fmt vet ## Build manager binary.
+	go build -o bin/manager main.go
+
+run: manifests generate fmt vet ## Run a controller from your host.
+	go run ./main.go --config config/manager/controller_manager_config.yaml
+
+docker-build: test ## Build docker image with the manager.
+	docker build -t ${IMG} .
+
+docker-push: ## Push docker image with the manager.
+	docker push ${IMG}
+
+##@ Deployment
+
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | kubectl delete -f -
+
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/[email protected])
+
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef

PROJECT

@@ -0,0 +1,25 @@
+domain: kubeflow.org
+layout:
+- go.kubebuilder.io/v3
+multigroup: true
+projectName: project
+repo: github.com/sky-uk/kfp-operator
+resources:
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: kubeflow.org
+  group: pipelines
+  kind: Pipeline
+  path: github.com/sky-uk/kfp-operator/api/v1
+  version: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kubeflow.org
+  group: config
+  kind: KfpControllerConfig
+  path: github.com/sky-uk/kfp-operator/apis/config/v1
+  version: v1
+version: "3"

README.md

@@ -0,0 +1,13 @@
+# Kubeflow Pipelines Operator
+
+## Overview
+
+The Kubefelow Pipelines Operator provides a declarative API for managing and running machine learning pipelines on Kubeflow with [Resource Definitions](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/).
+
+We currently only support TFX pipelines.
+
+## Documentation:
+
+- [Documentation](docs)
+- [Quickstart](docs/quickstart)
+- [Development Guide](docs/development)

apis/config/v1/groupversion_info.go

@@ -0,0 +1,20 @@
+// Package v2 contains API Schema definitions for the config v2 API group
+//+kubebuilder:object:generate=true
+//+groupName=config.kubeflow.org
+package v2
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "config.kubeflow.org", Version: "v1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

apis/config/v1/kfp_controller_config_types.go

@@ -0,0 +1,33 @@
+package v2
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
+)
+
+type Configuration struct {
+	PipelineStorage string `json:"pipelineStorage,omitempty"`
+	DataflowProject string `json:"dataflowProject,omitempty"`
+	ImagePullPolicy string `json:"imagePullPolicy,omitempty"`
+	ServiceAccount  string `json:"serviceAccount,omitempty"`
+	KfpEndpoint     string `json:"kfpEndpoint,omitempty"`
+	CompilerImage   string `json:"compilerImage,omitempty"`
+	KfpToolsImage   string `json:"kfpToolsImage,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+type KfpControllerConfig struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Workflows         Configuration `json:"spec,omitempty"`
+
+	cfg.ControllerManagerConfigurationSpec `json:"controller,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+func init() {
+	SchemeBuilder.Register(&KfpControllerConfig{})
+}

apis/pipelines/v1/groupversion_info.go

@@ -0,0 +1,20 @@
+// Package v1 contains API Schema definitions for the pipelines.kubeflow.org v1 API group
+//+kubebuilder:object:generate=true
+//+groupName=pipelines.kubeflow.org
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "pipelines.kubeflow.org", Version: "v1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)