[occm] Support re-use load balancer between Services (kubernetes#1648)

By default, different Services of LoadBalancer type should have
different corresponding cloud load balancers, however,
openstack-cloud-controller-manager allows multiple Services to share a
single load balancer if the Octavia service supports the tag feature
(since version 2.5).

Added a new annotation `loadbalancer.openstack.org/load-balancer-id`
that can be specified for creating Services.

Added a new config option `[LoadBalancer] max-shared-lb` to limit the
number of Services that share a load balancer.

Check the doc for more details.

Makefile

@@ -23,7 +23,6 @@ DEST := $(GOPATH)/src/$(GIT_HOST)/$(BASE_DIR)
 SOURCES := $(shell find $(DEST) -name '*.go' 2>/dev/null)
 HAS_LINT := $(shell command -v golint;)
 HAS_GOX := $(shell command -v gox;)
-HAS_IMPORT_BOSS := $(shell command -v import-boss;)
 GOX_PARALLEL ?= 3
 
 TARGETS		?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64le linux/s390x
@@ -156,7 +155,7 @@ build-cmd-%: work $(SOURCES)
 
 test: unit functional
 
-check: work fmt vet lint import-boss
+check: work fmt vet lint
 
 unit: work
 	go test -tags=unit $(shell go list ./... | sed -e '/sanity/ { N; d; }' | sed -e '/tests/ {N; d;}') $(TESTARGS)
@@ -176,17 +175,10 @@ fmt:
 lint:
 ifndef HAS_LINT
 	echo "installing lint"
-	GO111MODULE=off go get -u golang.org/x/lint/golint
+	go get -u golang.org/x/lint/golint
 endif
 	hack/verify-golint.sh
 
-import-boss:
-ifndef HAS_IMPORT_BOSS
-	echo "installing import-boss"
-	go install k8s.io/code-generator/cmd/[email protected]
-endif
-	hack/verify-import-boss.sh
-
 vet:
 	go vet ./...
 
@@ -333,5 +325,5 @@ dist: build-cross
 		$(DIST_DIRS) zip -r cloud-provider-openstack-$(VERSION)-{}.zip {} \; \
 	)
 
-.PHONY: bindep build clean cover work docs fmt functional lint import-boss realclean \
+.PHONY: bindep build clean cover work docs fmt functional lint realclean \
 	relnotes test translation version build-cross dist

docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md

@@ -10,6 +10,7 @@
     - [Creating Service by specifying a floating IP](#creating-service-by-specifying-a-floating-ip)
     - [Restrict Access For LoadBalancer Service](#restrict-access-for-loadbalancer-service)
     - [Use PROXY protocol to preserve client IP](#use-proxy-protocol-to-preserve-client-ip)
+    - [Sharing load balancer with multiple Services](#sharing-load-balancer-with-multiple-services)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -101,7 +102,7 @@ Request Body:
 - `loadbalancer.openstack.org/floating-subnet-tags`
 
   This annotation is the tag of a subnet belonging to the floating network.
-  
+
 - `loadbalancer.openstack.org/class`
 
   The name of a preconfigured class in the config file. If provided, this config options included in the class section take precedence over the annotations of floating-subnet-id and floating-network-id. See the section below for how it works.
@@ -170,10 +171,10 @@ Request Body:
 
 - `loadbalancer.openstack.org/enable-health-monitor`
 
-  Defines whether or not to create health monitor for the load balancer pool, if not specified, use `create-monitor` config. The health monitor can be created or deleted dynamically.
+  Defines whether to create health monitor for the load balancer pool, if not specified, use `create-monitor` config. The health monitor can be created or deleted dynamically.
 
   Not supported when `lb-provider=ovn` is configured in openstack-cloud-controller-manager.
-  
+
 - `loadbalancer.openstack.org/flavor-id`
 
   The id of the flavor that is used for creating the loadbalancer.
@@ -193,6 +194,14 @@ Request Body:
 
   Not supported when `lb-provider=ovn` is configured in openstack-cloud-controller-manager.
 
+- `loadbalancer.openstack.org/load-balancer-id`
+
+  This annotation is automatically added to the Service if it's not specified when creating. After the Service is created successfully it shouldn't be changed, otherwise the Service won't behave as expected.  
+
+  If this annotation is specified with a valid cloud load balancer ID when creating Service, the Service is reusing this load balancer rather than creating another one. Again, it shouldn't be changed after the Service is created.
+
+  If this annotation is specified, the other annotations which define the load balancer features will be ignored.
+
 ### Switching between Floating Subnets by using preconfigured Classes
 
 If you have multiple `FloatingIPPools` and/or `FloatingIPSubnets` it might be desirable to offer the user logical meanings for `LoadBalancers` like `internetFacing` or `DMZ` instead of requiring the user to select a dedicated network or subnet ID at the service object level as an annotation.
@@ -232,7 +241,7 @@ floating-subnet-id="b374bed4-e920-4c40-b646-2d8927f7f67b"
 
 Within a `LoadBalancerClass` one of `floating-subnet-id`, `floating-subnet` or `floating-subnet-tags` is mandatory.
 `floating-subnet-id` takes precedence over the other ones with must all match if specified.
-If the pattern starts with a `!`, the match is negated. 
+If the pattern starts with a `!`, the match is negated.
 The rest of the pattern can either be a direct name, a glob or a regular expression if it starts with a `~`.
 `floating-subnet-tags` can be a comma separated list of tags. By default it matches a subnet if at least one tag is present.
 If the list is preceded by a `&` all tags must be present. Again with a preceding `!` the condition be be negated.
@@ -450,3 +459,110 @@ To enable PROXY protocol support, the openstack-cloud-controller-manager config
    Request Body:
            -no body in request-
    ```
+
+### Sharing load balancer with multiple Services
+
+By default, different Services of LoadBalancer type should have different corresponding cloud load balancers, however, openstack-cloud-controller-manager allows multiple Services to share a single load balancer if the Octavia service supports the tag feature (since version 2.5).
+
+The shared load balancer can be created either by other Services or outside the cluster, e.g. created manually by the user in the cloud or by Services from the other Kubernetes clusters. The load balancer is deleted only when the last attached Service is deleted, unless the load balancer was created outside the Kubernetes cluster.
+
+The maximum number of Services that share a load balancer can be configured in `[LoadBalancer] max-shared-lb`, default value is 2. The ports of those Services shouldn't have collisions.
+
+For example, create a Service `service-1` as before:
+
+```yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: service-1
+  namespace: default
+spec:
+  type: LoadBalancer
+  selector:
+    app: webserver
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+```
+
+When `service-1` is created successfully, check the load balancer created in the cloud, which has its name in its tags.
+
+```shell
+$ openstack loadbalancer show 2b224530-9414-4302-8163-5abebdcdc84f -c name -c tags
++-------+---------------------------------------------+
+| Field | Value                                       |
++-------+---------------------------------------------+
+| name  | kube_service_cluster-name_default_service-1 |
+| tags  | kube_service_cluster-name_default_service-1 |
++-------+---------------------------------------------+
+```
+
+Check the Service, you should notice a new annotation `loadbalancer.openstack.org/load-balancer-id` is added:
+
+```shell
+$ kubectl describe service service-1 | grep loadbalancer.openstack.org/load-balancer-id
+                          loadbalancer.openstack.org/load-balancer-id: 2b224530-9414-4302-8163-5abebdcdc84f
+```
+
+> NOTE: Do not update the annotation `loadbalancer.openstack.org/load-balancer-id` after the Service is created successfully or the relationship between Service and the load balancer will be broken.
+
+Now, create another Service `service-2` but re-use the load balancer created for `service-1` by specifying the annotation `loadbalancer.openstack.org/load-balancer-id`:
+
+```yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: service-2
+  namespace: default
+  annotations:
+    loadbalancer.openstack.org/load-balancer-id: "2b224530-9414-4302-8163-5abebdcdc84f"
+spec:
+  type: LoadBalancer
+  selector:
+    app: webserver
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+```
+
+After `service-2` is created successfully, check the load balancer again, you'll see there is a new tag added. Now the load balancer should have 2 listeners, listening on the ports of the 2 Services respectively.
+
+```shell
+$ openstack loadbalancer show 2b224530-9414-4302-8163-5abebdcdc84f -c name -c tags
++-------+---------------------------------------------+
+| Field | Value                                       |
++-------+---------------------------------------------+
+| name  | kube_service_lingxian-k8s_default_service-1 |
+| tags  | kube_service_lingxian-k8s_default_service-1 |
+|       | kube_service_lingxian-k8s_default_service-2 |
++-------+---------------------------------------------+
+$ openstack loadbalancer listener list --loadbalancer 2b224530-9414-4302-8163-5abebdcdc84f -c id -c protocol -c protocol_port
++--------------------------------------+----------+---------------+
+| id                                   | protocol | protocol_port |
++--------------------------------------+----------+---------------+
+| 05fbcc93-61e5-4eb4-be21-632ab8022d46 | TCP      |            80 |
+| 50e94cc4-f08e-4c71-9ee4-4488350834f6 | TCP      |          8080 |
++--------------------------------------+----------+---------------+
+```
+
+Check the load balancer again after deleting `service-1`:
+
+```shell
+$ openstack loadbalancer show 2b224530-9414-4302-8163-5abebdcdc84f -c name -c tags
++-------+---------------------------------------------+
+| Field | Value                                       |
++-------+---------------------------------------------+
+| name  | kube_service_lingxian-k8s_default_service-1 |
+| tags  | kube_service_lingxian-k8s_default_service-2 |
++-------+---------------------------------------------+
+$ openstack loadbalancer listener list --loadbalancer 2b224530-9414-4302-8163-5abebdcdc84f -c id -c protocol -c protocol_port
++--------------------------------------+----------+---------------+
+| id                                   | protocol | protocol_port |
++--------------------------------------+----------+---------------+
+| 50e94cc4-f08e-4c71-9ee4-4488350834f6 | TCP      |          8080 |
++--------------------------------------+----------+---------------+
+```
+
+The load balancer will be deleted after `service-2` is deleted. 

docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md

@@ -253,6 +253,9 @@ Although the openstack-cloud-controller-manager was initially implemented with N
 
   Format for tls container ref: `https://{keymanager_host}/v1/containers/{uuid}`
 
+* `max-shared-lb`
+  The maximum number of Services that share a load balancer. Default: 2
+
 NOTE:
 
 * When using `ovn` provider service has limited scope - `create_monitor` is not supported and only supported `lb-method` is `SOURCE_IP`.

go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 // indirect
 	github.com/container-storage-interface/spec v1.5.0
 	github.com/golang/protobuf v1.5.2
-	github.com/gophercloud/gophercloud v0.19.0
-	github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d
+	github.com/gophercloud/gophercloud v0.21.0
+	github.com/gophercloud/utils v0.0.0-20210909165623-d7085207ff6d
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-version v1.2.0
 	github.com/imdario/mergo v0.3.7 // indirect

go.sum

@@ -289,11 +289,11 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gophercloud/gophercloud v0.6.1-0.20191122030953-d8ac278c1c9d/go.mod h1:ozGNgr9KYOVATV5jsgHl/ceCDXGuguqOZAzoQ/2vcNM=
-github.com/gophercloud/gophercloud v0.19.0 h1:zzaIh8W2K5M4AkJhPV1z6O4Sp0FOObzXm61NUmFz3Kw=
-github.com/gophercloud/gophercloud v0.19.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
-github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d h1:fduaPzWwIfvOMLuHk2Al3GZH0XbUqG8MbElPop+Igzs=
-github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d/go.mod h1:ehWUbLQJPqS0Ep+CxeD559hsm9pthPXadJNKwZkp43w=
+github.com/gophercloud/gophercloud v0.20.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
+github.com/gophercloud/gophercloud v0.21.0 h1:21rxoQM7cSaZCPgfP45h71Vt1amZa942l7AtUWLOI2I=
+github.com/gophercloud/gophercloud v0.21.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
+github.com/gophercloud/utils v0.0.0-20210909165623-d7085207ff6d h1:0Wsi5dvUuPF6dVn/CNfEA4xLxmaEtOt7tV2HD16xIf8=
+github.com/gophercloud/utils v0.0.0-20210909165623-d7085207ff6d/go.mod h1:qOGlfG6OIJ193/c3Xt/XjOfHataNZdQcVgiu93LxBUM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
 github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -324,6 +324,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=

pkg/ingress/controller/controller.go

@@ -592,7 +592,7 @@ func (c *Controller) deleteIngress(ing *nwv1.Ingress) error {
 		logger.Info("security group deleted")
 	}
 
-	err = openstackutil.DeleteLoadbalancer(c.osClient.Octavia, loadbalancer.ID)
+	err = openstackutil.DeleteLoadbalancer(c.osClient.Octavia, loadbalancer.ID, true)
 	logger.WithFields(log.Fields{"lbID": loadbalancer.ID}).Info("loadbalancer deleted")
 
 	return err