Merge pull request #528 from SchSeba/ci_improvements

Support number of workers for the virtual cluster

Makefile

@@ -197,6 +197,9 @@ test-e2e-conformance-virtual-ocp-cluster-ci:
 test-e2e-conformance-virtual-ocp-cluster:
 	SKIP_DELETE=TRUE ./hack/run-e2e-conformance-virtual-ocp.sh
 
+redeploy-operator-virtual-cluster:
+	./hack/virtual-cluster-redeploy.sh
+
 test-e2e-validation-only:
 	SUITE=./test/validation ./hack/run-e2e-conformance.sh	
 

doc/testing-virtual-machine.md

@@ -35,6 +35,8 @@ WantedBy=network-online.target
 * make
 * go
 
+*Note:* For openshift you will also need a pull secret that you can download from the [Redhat Console](https://console.redhat.com/)
+
 ## Deploy the cluster
 
 use the deployment [script](../hack/run-e2e-conformance-virtual-cluster.sh), this will deploy a k8s cluster
@@ -55,4 +57,12 @@ To use the cluster after the deployment you need to export the kubeconfig
 
 ```
 export KUBECONFIG=$HOME/.kcli/clusters/virtual/auth/kubeconfig
-```
+```
+
+It's possible to also configure the number of workers using `NUM_OF_WORKERS`.
+
+*NOTE:* For OCP then min number is 3 and for k8s is 2.
+
+To work on the operator you can change the code and rebuild the operator using
+`make redeploy-operator-virtual-cluster`.
+You need to tell the cluster type for ocp `CLUSTER_TYPE=openshift` and for k8s `CLUSTER_TYPE=kubernetes`

hack/run-e2e-conformance-virtual-cluster.sh

@@ -11,6 +11,14 @@ HOME="/root"
 here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")"
 root="$(readlink --canonicalize "$here/..")"
 
+NUM_OF_WORKERS=${NUM_OF_WORKERS:-2}
+total_number_of_nodes=$((1 + NUM_OF_WORKERS))
+
+if [ "$NUM_OF_WORKERS" -lt 2 ]; then
+    echo "Min number of workers is 2"
+    exit 1
+fi
+
 check_requirements() {
   for cmd in kcli virsh virt-edit podman make go; do
     if ! command -v "$cmd" &> /dev/null; then
@@ -49,7 +57,7 @@ api_ip: $api_ip
 virtual_router_id: $virtual_router_id
 domain: $domain_name
 ctlplanes: 1
-workers: 2
+workers: $NUM_OF_WORKERS
 ingress: false
 machine: q35
 engine: crio
@@ -92,7 +100,7 @@ sleep_time=10
 until $ready || [ $ATTEMPTS -eq $MAX_ATTEMPTS ]
 do
     echo "waiting for cluster to be ready"
-    if [ `kubectl get node | grep Ready | wc -l` == 3 ]; then
+    if [ `kubectl get node | grep Ready | wc -l` == $total_number_of_nodes ]; then
         echo "cluster is ready"
         ready=true
     else
@@ -108,13 +116,21 @@ if ! $ready; then
     exit 1
 fi
 
+function update_worker_labels() {
 echo "## label cluster workers as sriov capable"
-kubectl label node $cluster_name-worker-0.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
-kubectl label node $cluster_name-worker-1.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
+for ((num=0; num<NUM_OF_WORKERS; num++))
+do
+    kubectl label node $cluster_name-worker-$num.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
+done
 
 echo "## label cluster worker as worker"
-kubectl label node $cluster_name-worker-0.$domain_name node-role.kubernetes.io/worker= --overwrite
-kubectl label node $cluster_name-worker-1.$domain_name node-role.kubernetes.io/worker= --overwrite
+for ((num=0; num<NUM_OF_WORKERS; num++))
+do
+  kubectl label node $cluster_name-worker-$num.$domain_name node-role.kubernetes.io/worker= --overwrite
+done
+}
+
+update_worker_labels
 
 controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'`
 insecure_registry="[[registry]]
@@ -157,9 +173,10 @@ EOF
 }
 
 update_host $cluster_name-ctlplane-0
-update_host $cluster_name-worker-0
-update_host $cluster_name-worker-1
-
+for ((num=0; num<NUM_OF_WORKERS; num++))
+do
+  update_host $cluster_name-worker-$num
+done
 
 kubectl create namespace container-registry
 

hack/run-e2e-conformance-virtual-ocp.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -xeo pipefail
 
+OCP_VERSION=${OCP_VERSION:-4.14.0-rc.6}
 cluster_name=${CLUSTER_NAME:-ocp-virt}
 domain_name=lab
 
@@ -9,6 +10,14 @@ virtual_router_id=${VIRTUAL_ROUTER_ID:-253}
 registry="default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}"
 HOME="/root"
 
+NUM_OF_WORKERS=${NUM_OF_WORKERS:-3}
+total_number_of_nodes=$((1 + NUM_OF_WORKERS))
+
+if [ "$NUM_OF_WORKERS" -lt 3 ]; then
+    echo "Min number of workers is 3"
+    exit 1
+fi
+
 here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")"
 root="$(readlink --canonicalize "$here/..")"
 
@@ -41,7 +50,7 @@ kcli create network -c 192.168.123.0/24 ocp
 kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name
 
 cat <<EOF > ./${cluster_name}-plan.yaml
-tag: 4.14.0-rc.6
+tag: $OCP_VERSION
 ctlplane_memory: 32768
 worker_memory: 8192
 pool: default
@@ -51,7 +60,7 @@ api_ip: $api_ip
 virtual_router_id: $virtual_router_id
 domain: $domain_name
 ctlplanes: 1
-workers: 3
+workers: $NUM_OF_WORKERS
 machine: q35
 network_type: OVNKubernetes
 pull_secret: /root/openshift_pull.json
@@ -97,7 +106,7 @@ sleep_time=10
 until $ready || [ $ATTEMPTS -eq $MAX_ATTEMPTS ]
 do
     echo "waiting for cluster to be ready"
-    if [ `kubectl get node | grep Ready | wc -l` == 4 ]; then
+    if [ `kubectl get node | grep Ready | wc -l` == $total_number_of_nodes ]; then
         echo "cluster is ready"
         ready=true
     else
@@ -114,9 +123,10 @@ if ! $ready; then
 fi
 
 echo "## label cluster workers as sriov capable"
-kubectl label node $cluster_name-worker-0.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
-kubectl label node $cluster_name-worker-1.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
-kubectl label node $cluster_name-worker-2.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
+for ((num=0; num<NUM_OF_WORKERS; num++))
+do
+    kubectl label node $cluster_name-worker-$num.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite
+done
 
 controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'`
 

hack/virtual-cluster-redeploy.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -xeo pipefail
+
+HOME="/root"
+here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")"
+root="$(readlink --canonicalize "$here/..")"
+domain_name=lab
+
+if [ $CLUSTER_TYPE == "openshift" ]; then
+  echo "Openshift"
+  cluster_name=${CLUSTER_NAME:-ocp-virt}
+  export NAMESPACE="openshift-sriov-network-operator"
+  export KUBECONFIG=/root/.kcli/clusters/$cluster_name/auth/kubeconfig
+
+  dockercgf=`kubectl -n ${NAMESPACE} get sa builder -oyaml | grep imagePullSecrets -A 1 | grep -o "builder-.*"`
+  auth=`kubectl -n ${NAMESPACE} get secret ${dockercgf} -ojson | jq '.data.".dockercfg"'`
+  auth="${auth:1:-1}"
+  auth=`echo ${auth} | base64 -d`
+  echo ${auth} > registry-login.conf
+
+  internal_registry="image-registry.openshift-image-registry.svc:5000"
+  pass=$( jq .\"$internal_registry\".password registry-login.conf )
+
+  registry="default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}"
+  podman login -u serviceaccount -p ${pass:1:-1} $registry --tls-verify=false
+
+  export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest"
+  export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest"
+  export SRIOV_NETWORK_WEBHOOK_IMAGE="$registry/$NAMESPACE/sriov-network-operator-webhook:latest"
+else
+  echo "K8S"
+  cluster_name=${CLUSTER_NAME:-virtual}
+  export NAMESPACE="sriov-network-operator"
+  export KUBECONFIG=/root/.kcli/clusters/$cluster_name/auth/kubeconfig
+
+  controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'`
+
+  export SRIOV_NETWORK_OPERATOR_IMAGE="$controller_ip:5000/sriov-network-operator:latest"
+  export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$controller_ip:5000/sriov-network-config-daemon:latest"
+  export SRIOV_NETWORK_WEBHOOK_IMAGE="$controller_ip:5000/sriov-network-operator-webhook:latest"
+fi
+
+export ENABLE_ADMISSION_CONTROLLER=true
+export SKIP_VAR_SET=""
+export OPERATOR_NAMESPACE=$NAMESPACE
+export OPERATOR_EXEC=kubectl
+export DEV_MODE=TRUE
+export CLUSTER_HAS_EMULATED_PF=TRUE
+
+echo "## build operator image"
+podman build -t "${SRIOV_NETWORK_OPERATOR_IMAGE}" -f "${root}/Dockerfile" "${root}"
+
+echo "## build daemon image"
+podman build -t "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}" -f "${root}/Dockerfile.sriov-network-config-daemon" "${root}"
+
+echo "## build webhook image"
+podman build -t "${SRIOV_NETWORK_WEBHOOK_IMAGE}" -f "${root}/Dockerfile.webhook" "${root}"
+
+podman push --tls-verify=false "${SRIOV_NETWORK_OPERATOR_IMAGE}"
+podman push --tls-verify=false "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}"
+podman push --tls-verify=false "${SRIOV_NETWORK_WEBHOOK_IMAGE}"
+
+if [ $CLUSTER_TYPE == "openshift" ]; then
+  export SRIOV_NETWORK_OPERATOR_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator:latest"
+  export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-config-daemon:latest"
+  export SRIOV_NETWORK_WEBHOOK_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator-webhook:latest"
+fi
+
+echo "## deploying SRIOV Network Operator"
+hack/deploy-setup.sh $NAMESPACE
+
+kubectl -n ${NAMESPACE} delete po --all