Implement ResourceQuota (#617)

* Add `core` group
* Implement `core` `ResourceQuota` type
* Update all generative code
* Add registry / types / samples / validation
* Implement resource quota evaluator and registry
* Implement resource quota admission
* Implement resource quota controller and replenish controllers
* Create test cases
* Move apiserver internal directory to onmetal internal

.golangci.yaml

@@ -30,9 +30,7 @@ linters-settings:
     alias:
       - pkg: github.com/onmetal/onmetal-api/api/(\w+)/(v[\w\d]+)
         alias: $1$2
-      - pkg: github.com/onmetal/onmetal-api/onmetal-apiserver/internal/apis/(\w+)/(v[\w\d]+)
-        alias: $1$2
-      - pkg: github.com/onmetal/onmetal-api/onmetal-apiserver/internal/apis/(\w+)
+      - pkg: github.com/onmetal/onmetal-api/internal/apis/(\w+)
         alias: $1
 
 issues:

Dockerfile

@@ -13,6 +13,7 @@ RUN go mod download
 COPY api/ api/
 COPY broker/ broker/
 COPY client-go/ client-go/
+COPY internal/ internal/
 COPY onmetal-apiserver/ onmetal-apiserver/
 COPY onmetal-controller-manager/ onmetal-controller-manager/
 COPY ori/ ori/

Makefile

@@ -13,7 +13,7 @@ BUCKETBROKER_IMG ?= bucketbroker:latest
 ORICTL_BUCKET_IMG ?= orictl-bucket:latest
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.24.1
+ENVTEST_K8S_VERSION = 1.26.1
 
 # Docker image name for the mkdocs based local development setup
 IMAGE=onmetal-api/documentation
@@ -128,6 +128,7 @@ check: manifests generate add-license lint test # Generate manifests, code, lint
 .PHONY: docs
 docs: gen-crd-api-reference-docs ## Run go generate to generate API reference documentation.
 	$(GEN_CRD_API_REFERENCE_DOCS) -api-dir ./api/common/v1alpha1 -config ./hack/api-reference/common-config.json -template-dir ./hack/api-reference/template -out-file ./docs/api-reference/common.md
+	$(GEN_CRD_API_REFERENCE_DOCS) -api-dir ./api/core/v1alpha1 -config ./hack/api-reference/core-config.json -template-dir ./hack/api-reference/template -out-file ./docs/api-reference/core.md
 	$(GEN_CRD_API_REFERENCE_DOCS) -api-dir ./api/compute/v1alpha1 -config ./hack/api-reference/compute-config.json -template-dir ./hack/api-reference/template -out-file ./docs/api-reference/compute.md
 	$(GEN_CRD_API_REFERENCE_DOCS) -api-dir ./api/storage/v1alpha1 -config ./hack/api-reference/storage-config.json -template-dir ./hack/api-reference/template -out-file ./docs/api-reference/storage.md
 	$(GEN_CRD_API_REFERENCE_DOCS) -api-dir ./api/networking/v1alpha1 -config ./hack/api-reference/networking-config.json -template-dir ./hack/api-reference/template -out-file ./docs/api-reference/networking.md
@@ -150,9 +151,13 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test-only: envtest ## Run *only* the tests - no generation, linting etc.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
 
-.PHONEY: openapi-extractor
-extract-openapi: openapi-extractor
-	$(OPENAPI_EXTRACTOR) --apiserver-package="github.com/onmetal/onmetal-api/onmetal-apiserver/cmd/apiserver" --apiserver-build-opts=mod --apiservices="./config/apiserver/apiservice/bases" --output="./gen"
+.PHONY: openapi-extractor
+extract-openapi: envtest openapi-extractor
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(OPENAPI_EXTRACTOR) \
+		--apiserver-package="github.com/onmetal/onmetal-api/onmetal-apiserver/cmd/apiserver" \
+		--apiserver-build-opts=mod \
+		--apiservices="./config/apiserver/apiservice/bases" \
+		--output="./gen"
 
 ##@ Build
 

api/core/v1alpha1/doc.go

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2021 by the OnMetal authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// +k8s:deepcopy-gen=package
+// +k8s:openapi-gen=true
+// +groupName=core.api.onmetal.de
+
+// Package v1alpha1 is the v1alpha1 version of the API.
+package v1alpha1 // import "github.com/onmetal/onmetal-api/api/core/v1alpha1"

api/core/v1alpha1/register.go

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2021 by the OnMetal authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package v1alpha1 contains API Schema definitions for the common v1alpha1 API group
+// +groupName=core.api.onmetal.de
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	// SchemeGroupVersion is group version used to register these objects
+	SchemeGroupVersion = schema.GroupVersion{Group: "core.api.onmetal.de", Version: "v1alpha1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&ResourceQuota{},
+		&ResourceQuotaList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

api/core/v1alpha1/resource.go

@@ -0,0 +1,79 @@
+// Copyright 2023 OnMetal authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// ResourceName is the name of a resource, most often used alongside a resource.Quantity.
+type ResourceName string
+
+const (
+	// ResourceCPU is the amount of cpu in cores.
+	ResourceCPU ResourceName = "cpu"
+	// ResourceMemory is the amount of memory in bytes.
+	ResourceMemory ResourceName = "memory"
+	// ResourceStorage is the amount of storage, in bytes.
+	ResourceStorage ResourceName = "storage"
+
+	// ResourcesRequestsPrefix is the prefix used for limiting resource requests in ResourceQuota.
+	ResourcesRequestsPrefix = "requests."
+
+	// ResourceRequestsCPU is the amount of requested cpu in cores.
+	ResourceRequestsCPU = ResourcesRequestsPrefix + ResourceCPU
+	// ResourceRequestsMemory is the amount of requested memory in bytes.
+	ResourceRequestsMemory = ResourcesRequestsPrefix + ResourceMemory
+	// ResourceRequestsStorage is the amount of requested storage in bytes.
+	ResourceRequestsStorage = ResourcesRequestsPrefix + ResourceStorage
+
+	// ResourceCountNamespacePrefix is resource namespace prefix for counting resources.
+	ResourceCountNamespacePrefix = "count/"
+)
+
+// ObjectCountQuotaResourceNameFor returns the ResourceName for counting the given groupResource.
+func ObjectCountQuotaResourceNameFor(groupResource schema.GroupResource) ResourceName {
+	if len(groupResource.Group) == 0 {
+		return ResourceName("count/" + groupResource.Resource)
+	}
+	return ResourceName(ResourceCountNamespacePrefix + groupResource.Resource + "." + groupResource.Group)
+}
+
+// ResourceList is a list of ResourceName alongside their resource.Quantity.
+type ResourceList map[ResourceName]resource.Quantity
+
+// Name returns the resource with name if specified, otherwise it returns a nil quantity with default format.
+func (rl *ResourceList) Name(name ResourceName, defaultFormat resource.Format) *resource.Quantity {
+	if val, ok := (*rl)[name]; ok {
+		return &val
+	}
+	return &resource.Quantity{Format: defaultFormat}
+}
+
+// Storage is a shorthand for getting the quantity associated with ResourceStorage.
+func (rl *ResourceList) Storage() *resource.Quantity {
+	return rl.Name(ResourceStorage, resource.BinarySI)
+}
+
+// Memory is a shorthand for getting the quantity associated with ResourceMemory.
+func (rl *ResourceList) Memory() *resource.Quantity {
+	return rl.Name(ResourceMemory, resource.BinarySI)
+}
+
+// CPU is a shorthand for getting the quantity associated with ResourceCPU.
+func (rl *ResourceList) CPU() *resource.Quantity {
+	return rl.Name(ResourceCPU, resource.DecimalSI)
+}

api/core/v1alpha1/resourcequota_types.go

@@ -0,0 +1,95 @@
+// Copyright 2023 OnMetal authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ResourceQuotaSpec defines the desired state of ResourceQuotaSpec
+type ResourceQuotaSpec struct {
+	// Hard is a ResourceList of the strictly enforced amount of resources.
+	Hard ResourceList `json:"hard,omitempty"`
+	// ScopeSelector selects the resources that are subject to this quota.
+	// Note: By using certain ScopeSelectors, only certain resources may be tracked.
+	ScopeSelector *ResourceScopeSelector `json:"scopeSelector,omitempty"`
+}
+
+// ResourceScopeSelector selects
+type ResourceScopeSelector struct {
+	// MatchExpressions is a list of ResourceScopeSelectorRequirement to match resources by.
+	MatchExpressions []ResourceScopeSelectorRequirement `json:"matchExpressions,omitempty"`
+}
+
+// ResourceScope is a scope of a resource.
+type ResourceScope string
+
+const (
+	// ResourceScopeMachineClass refers to the machine class of a resource.
+	ResourceScopeMachineClass ResourceScope = "MachineClass"
+	// ResourceScopeVolumeClass refers to the volume class of a resource.
+	ResourceScopeVolumeClass ResourceScope = "VolumeClass"
+)
+
+// ResourceScopeSelectorOperator is an operator to compare a ResourceScope with values.
+type ResourceScopeSelectorOperator string
+
+const (
+	ResourceScopeSelectorOperatorIn           ResourceScopeSelectorOperator = "In"
+	ResourceScopeSelectorOperatorNotIn        ResourceScopeSelectorOperator = "NotIn"
+	ResourceScopeSelectorOperatorExists       ResourceScopeSelectorOperator = "Exists"
+	ResourceScopeSelectorOperatorDoesNotExist ResourceScopeSelectorOperator = "DoesNotExist"
+)
+
+// ResourceScopeSelectorRequirement is a requirement for a resource using a ResourceScope alongside
+// a ResourceScopeSelectorOperator with Values (depending on the ResourceScopeSelectorOperator).
+type ResourceScopeSelectorRequirement struct {
+	// ScopeName is the ResourceScope to make a requirement for.
+	ScopeName ResourceScope `json:"scopeName"`
+	// Operator is the ResourceScopeSelectorOperator to check the ScopeName with in a resource.
+	Operator ResourceScopeSelectorOperator `json:"operator"`
+	// Values are the values to compare the Operator with the ScopeName. May be optional.
+	Values []string `json:"values,omitempty"`
+}
+
+// ResourceQuotaStatus is the status of a ResourceQuota.
+type ResourceQuotaStatus struct {
+	// Hard are the currently enforced hard resource limits. Hard may be less than used in
+	// case the limits were introduced / updated after more than allowed resources were already present.
+	Hard ResourceList `json:"hard,omitempty"`
+	// Used is the amount of currently used resources.
+	Used ResourceList `json:"used,omitempty"`
+}
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// ResourceQuota is the Schema for the resourcequotas API
+type ResourceQuota struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ResourceQuotaSpec   `json:"spec,omitempty"`
+	Status ResourceQuotaStatus `json:"status,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// ResourceQuotaList contains a list of ResourceQuota
+type ResourceQuotaList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ResourceQuota `json:"items"`
+}