From a8cbbb72d2906cc6179683b82d7f4f8f87e5b1df Mon Sep 17 00:00:00 2001 From: Bracha Frenkel Date: Sun, 12 May 2024 17:30:34 +0300 Subject: [PATCH] [Kuttl] Add test for OVN db cluster pod deletions test steps: 1. stand up a 3 replicas ovn db cluster (both nb & sb). 2. confirm pods are all up. 3. confirm that the pods established the mesh. 4. delete all pods. 5. check that new pods are respawned. 6. check that they re-established the mesh with the correct number of pods. related Jira: OSPRH-6135 --- .../common/scripts/check_cluster_status.sh | 53 +++++++++++++++++++ .../kuttl/tests/ovn_db_delete/01-assert.yaml | 1 + .../tests/ovn_db_delete/01-deploy-ovn.yaml | 1 + .../kuttl/tests/ovn_db_delete/02-assert.yaml | 47 ++++++++++++++++ .../tests/ovn_db_delete/02-scale-ovndb.yaml | 7 +++ .../kuttl/tests/ovn_db_delete/03-assert.yaml | 9 ++++ .../kuttl/tests/ovn_db_delete/04-assert.yaml | 1 + .../tests/ovn_db_delete/04-delete-pods.yaml | 7 +++ .../kuttl/tests/ovn_db_delete/05-assert.yaml | 1 + .../kuttl/tests/ovn_db_delete/06-assert.yaml | 1 + .../06-delete-pods-non-graceful.yaml | 7 +++ .../kuttl/tests/ovn_db_delete/07-assert.yaml | 1 + .../kuttl/tests/ovn_db_delete/08-assert.yaml | 48 +++++++++++++++++ .../ovn_db_delete/08-scale-down-ovndb.yaml | 7 +++ .../kuttl/tests/ovn_db_delete/09-assert.yaml | 9 ++++ .../tests/ovn_db_delete/10-cleanup-ovn.yaml | 1 + .../kuttl/tests/ovn_db_delete/10-errors.yaml | 1 + 17 files changed, 202 insertions(+) create mode 100755 tests/kuttl/common/scripts/check_cluster_status.sh create mode 120000 tests/kuttl/tests/ovn_db_delete/01-assert.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/01-deploy-ovn.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/02-assert.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/02-scale-ovndb.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/03-assert.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/04-assert.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/04-delete-pods.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/05-assert.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/06-assert.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/06-delete-pods-non-graceful.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/07-assert.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/08-assert.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/08-scale-down-ovndb.yaml create mode 100644 tests/kuttl/tests/ovn_db_delete/09-assert.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/10-cleanup-ovn.yaml create mode 120000 tests/kuttl/tests/ovn_db_delete/10-errors.yaml diff --git a/tests/kuttl/common/scripts/check_cluster_status.sh b/tests/kuttl/common/scripts/check_cluster_status.sh new file mode 100755 index 00000000..e062c44a --- /dev/null +++ b/tests/kuttl/common/scripts/check_cluster_status.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# arguments: db-type: {nb, sb}, num-pods +# Check arguments +if [ $# -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +DB_TYPE="$1" +NUM_PODS="$2" +POD_PREFIX="ovsdbserver-${DB_TYPE}" +CTL_FILE="ovn${DB_TYPE}_db.ctl" +if [ "$DB_TYPE" == "nb" ]; then + DB_NAME="OVN_Northbound" +elif [ "$DB_TYPE" == "sb" ]; then + DB_NAME="OVN_Southbound" +fi + +declare -a pods +for i in $(seq 0 $((NUM_PODS-1))); do + pods+=("${POD_PREFIX}-${i}") +done + +# check each pod replica +for pod in "${pods[@]}"; do + + echo "Checking status of $pod" + output=$(oc exec $pod -n $NAMESPACE -- bash -c "OVS_RUNDIR=/tmp ovs-appctl -t /tmp/$CTL_FILE cluster/status $DB_NAME") + + # Example of part of output string that needs parsing: + # Status: cluster member + # Connections: ->0000 ->5476 <-5476 <-36cf + # Disconnections: 0 + # Servers: + # 36cf (36cf at ssl:ovsdbserver-nb-0.ovsdbserver-nb.openstack.svc.cluster.local:6643) last msg 590 ms ago + # 85de (85de at ssl:ovsdbserver-nb-2.ovsdbserver-nb.openstack.svc.cluster.local:6643) (self) + # 5476 (5476 at ssl:ovsdbserver-nb-1.ovsdbserver-nb.openstack.svc.cluster.local:6643) last msg 12063993 ms ago + + # Check if the pod is a cluster member + is_cluster_member=$(echo "$output" | grep -q "Status: cluster member"; echo $?) + if [ $is_cluster_member -ne 0 ]; then + exit 1 + fi + + # check if the pod is connected with all other pods + for server in "${pods[@]}"; do + echo "Checking if $server is mentioned in the output" + if ! echo "$output" | grep -q "$server"; then + exit 1 + fi + done +done diff --git a/tests/kuttl/tests/ovn_db_delete/01-assert.yaml b/tests/kuttl/tests/ovn_db_delete/01-assert.yaml new file mode 120000 index 00000000..461654ea --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/01-assert.yaml @@ -0,0 +1 @@ +../../common/assert_sample_deployment.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/01-deploy-ovn.yaml b/tests/kuttl/tests/ovn_db_delete/01-deploy-ovn.yaml new file mode 120000 index 00000000..ec519de5 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/01-deploy-ovn.yaml @@ -0,0 +1 @@ +../../common/deploy_ovn.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/02-assert.yaml b/tests/kuttl/tests/ovn_db_delete/02-assert.yaml new file mode 100644 index 00000000..72c814ad --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/02-assert.yaml @@ -0,0 +1,47 @@ +# +# Check for: +# +# - 1 OVNDBCluster CR +# - 3 Pods for OVNDBCluster nb CR +# + +apiVersion: ovn.openstack.org/v1beta1 +kind: OVNDBCluster +metadata: + finalizers: + - OVNDBCluster + name: ovndbcluster-nb-sample +spec: + replicas: 3 +status: + readyCount: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovsdbserver-nb +spec: + replicas: 3 +status: + availableReplicas: 3 +--- +apiVersion: ovn.openstack.org/v1beta1 +kind: OVNDBCluster +metadata: + finalizers: + - OVNDBCluster + name: ovndbcluster-sb-sample +spec: + replicas: 3 +status: + readyCount: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovsdbserver-sb +spec: + replicas: 3 +status: + availableReplicas: 3 +--- diff --git a/tests/kuttl/tests/ovn_db_delete/02-scale-ovndb.yaml b/tests/kuttl/tests/ovn_db_delete/02-scale-ovndb.yaml new file mode 100644 index 00000000..b6f5bfb6 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/02-scale-ovndb.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-nb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":3}]' + - script: | + oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-sb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":3}]' diff --git a/tests/kuttl/tests/ovn_db_delete/03-assert.yaml b/tests/kuttl/tests/ovn_db_delete/03-assert.yaml new file mode 100644 index 00000000..ffcf4ce9 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/03-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 3 + test $? -eq 0 + - script: | + $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 3 + test $? -eq 0 diff --git a/tests/kuttl/tests/ovn_db_delete/04-assert.yaml b/tests/kuttl/tests/ovn_db_delete/04-assert.yaml new file mode 120000 index 00000000..a30aa910 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/04-assert.yaml @@ -0,0 +1 @@ +02-assert.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/04-delete-pods.yaml b/tests/kuttl/tests/ovn_db_delete/04-delete-pods.yaml new file mode 100644 index 00000000..cce88807 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/04-delete-pods.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc delete pods -n $NAMESPACE -l service=ovsdbserver-nb + - script: | + oc delete pods -n $NAMESPACE -l service=ovsdbserver-sb diff --git a/tests/kuttl/tests/ovn_db_delete/05-assert.yaml b/tests/kuttl/tests/ovn_db_delete/05-assert.yaml new file mode 120000 index 00000000..cf30bf5b --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/05-assert.yaml @@ -0,0 +1 @@ +03-assert.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/06-assert.yaml b/tests/kuttl/tests/ovn_db_delete/06-assert.yaml new file mode 120000 index 00000000..a30aa910 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/06-assert.yaml @@ -0,0 +1 @@ +02-assert.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/06-delete-pods-non-graceful.yaml b/tests/kuttl/tests/ovn_db_delete/06-delete-pods-non-graceful.yaml new file mode 100644 index 00000000..e6c6a794 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/06-delete-pods-non-graceful.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc delete pods -n $NAMESPACE -l service=ovsdbserver-nb --force --grace-period=0 + - script: | + oc delete pods -n $NAMESPACE -l service=ovsdbserver-sb --force --grace-period=0 diff --git a/tests/kuttl/tests/ovn_db_delete/07-assert.yaml b/tests/kuttl/tests/ovn_db_delete/07-assert.yaml new file mode 120000 index 00000000..cf30bf5b --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/07-assert.yaml @@ -0,0 +1 @@ +03-assert.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/08-assert.yaml b/tests/kuttl/tests/ovn_db_delete/08-assert.yaml new file mode 100644 index 00000000..dfed497c --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/08-assert.yaml @@ -0,0 +1,48 @@ +# +# Check for: +# +# - 1 OVNDBCluster CR +# - 1 Pod for OVNDBCluster nb CR +# - 1 Pod for OVNDBCluster sb CR +# + +apiVersion: ovn.openstack.org/v1beta1 +kind: OVNDBCluster +metadata: + finalizers: + - OVNDBCluster + name: ovndbcluster-nb-sample +spec: + replicas: 1 +status: + readyCount: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovsdbserver-nb +spec: + replicas: 1 +status: + availableReplicas: 1 +--- +apiVersion: ovn.openstack.org/v1beta1 +kind: OVNDBCluster +metadata: + finalizers: + - OVNDBCluster + name: ovndbcluster-sb-sample +spec: + replicas: 1 +status: + readyCount: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovsdbserver-sb +spec: + replicas: 1 +status: + availableReplicas: 1 +--- diff --git a/tests/kuttl/tests/ovn_db_delete/08-scale-down-ovndb.yaml b/tests/kuttl/tests/ovn_db_delete/08-scale-down-ovndb.yaml new file mode 100644 index 00000000..388c6e6f --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/08-scale-down-ovndb.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-nb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":1}]' + - script: | + oc patch OVNDBCluster -n $NAMESPACE ovndbcluster-sb-sample --type='json' -p='[{"op": "replace", "path": "/spec/replicas", "value":1}]' diff --git a/tests/kuttl/tests/ovn_db_delete/09-assert.yaml b/tests/kuttl/tests/ovn_db_delete/09-assert.yaml new file mode 100644 index 00000000..38a7c6c5 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/09-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh nb 1 + test $? -eq 0 + - script: | + $OVN_KUTTL_DIR/../common/scripts/check_cluster_status.sh sb 1 + test $? -eq 0 diff --git a/tests/kuttl/tests/ovn_db_delete/10-cleanup-ovn.yaml b/tests/kuttl/tests/ovn_db_delete/10-cleanup-ovn.yaml new file mode 120000 index 00000000..02360341 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/10-cleanup-ovn.yaml @@ -0,0 +1 @@ +../../common/cleanup-ovn.yaml \ No newline at end of file diff --git a/tests/kuttl/tests/ovn_db_delete/10-errors.yaml b/tests/kuttl/tests/ovn_db_delete/10-errors.yaml new file mode 120000 index 00000000..7314c155 --- /dev/null +++ b/tests/kuttl/tests/ovn_db_delete/10-errors.yaml @@ -0,0 +1 @@ +../../common/errors_cleanup_ovn.yaml \ No newline at end of file