diff --git a/.zuul.yaml b/.zuul.yaml index 6e392cef20..6a555c7dcf 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -15,24 +15,6 @@ - ^OWNERS$ - ^SECURITY_CONTACTS$ - ^.gitignore$ - cloud-provider-openstack-e2e-test-csi-cinder: - jobs: - - cloud-provider-openstack-e2e-test-csi-cinder: - files: - - cmd/cinder-csi-plugin/.* - - manifests/cinder-csi-plugin/.* - - pkg/csi/cinder/.* - - pkg/util/.* - - tests/e2e/csi/cinder/.* - - cmd/tests/.* - - go.mod$ - - go.sum$ - - Makefile$ - irrelevant-files: - - docs/.*$ - - ^.*\.md$ - - ^OWNERS$ - - ^SECURITY_CONTACTS$ cloud-provider-openstack-multinode-csi-migration-test: jobs: - cloud-provider-openstack-multinode-csi-migration-test: diff --git a/tests/ci-csi-cinder-e2e.sh b/tests/ci-csi-cinder-e2e.sh new file mode 100755 index 0000000000..aab04ebadc --- /dev/null +++ b/tests/ci-csi-cinder-e2e.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################## +# +# Desc: Entrypoint of CSI cinder e2e CI job +# +# This script may be invoked for different branches (first added in +# release-1.23). It is getting GCP credentials from boskos and provision +# a GCP VM, then run ansible for the rest tasks. +# +######################################################################## + +set -x +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 +# PULL_NUMBER and PULL_BASE_REF are Prow job environment variables +PR_NUMBER="${PULL_NUMBER:-}" +[[ -z $PR_NUMBER ]] && echo "PR_NUMBER is required" && exit 1 +PR_BRANCH="${PULL_BASE_REF:-master}" +CONFIG_ANSIBLE="${CONFIG_ANSIBLE:-"true"}" +RESOURCE_TYPE="${RESOURCE_TYPE:-"gce-project"}" +ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" +mkdir -p "${ARTIFACTS}/logs/devstack" + +cleanup() { + # stop boskos heartbeat + [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}" +} +trap cleanup EXIT + +python3 -m pip install requests ansible + +# If BOSKOS_HOST is set then acquire a resource of type ${RESOURCE_TYPE} from Boskos. +if [ -n "${BOSKOS_HOST:-}" ]; then + # Check out the account from Boskos and store the produced environment + # variables in a temporary file. + account_env_var_file="$(mktemp)" + python3 tests/scripts/boskos.py --get --resource-type="${RESOURCE_TYPE}" 1>"${account_env_var_file}" + checkout_account_status="${?}" + + # If the checkout process was a success then load the account's + # environment variables into this process. + # shellcheck disable=SC1090 + [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}" + + # Always remove the account environment variable file. It contains + # sensitive information. + rm -f "${account_env_var_file}" + + if [ ! "${checkout_account_status}" = "0" ]; then + echo "Failed to get account from boskos, type: ${RESOURCE_TYPE}" 1>&2 + exit "${checkout_account_status}" + fi + + # run the heart beat process to tell boskos that we are still + # using the checked out account periodically + python3 -u tests/scripts/boskos.py --heartbeat >> "${ARTIFACTS}/logs/boskos.log" 2>&1 & + HEART_BEAT_PID=$! +fi + +case "${RESOURCE_TYPE}" in +"gce-project") + . tests/scripts/create-gce-vm.sh + ;; +*) + echo "Unsupported resource type: ${RESOURCE_TYPE}" + exit 1 + ;; +esac + +# Config ansible. If Ansible is installed from pip or from source, +# we need to create the config file manually. +if [[ "$CONFIG_ANSIBLE" == "true" ]]; then + mkdir -p /etc/ansible + cat < /etc/ansible/ansible.cfg +[defaults] +private_key_file = ~/.ssh/google_compute_engine +host_key_checking = False +timeout = 30 +stdout_callback = debug +EOF +fi + +# Run ansible playbook on the CI host, e.g. a VM in GCP +# USERNAME and PUBLIC_IP are global env variables set after creating the CI host. +ansible-playbook -v \ + --user ${USERNAME} \ + --private-key ~/.ssh/google_compute_engine \ + --inventory ${PUBLIC_IP}, \ + --ssh-common-args "-o StrictHostKeyChecking=no" \ + tests/playbooks/test-csi-cinder-e2e.yaml \ + -e github_pr=${PR_NUMBER} \ + -e github_pr_branch=${PR_BRANCH} +exit_code=$? + +# Fetch devstack logs for debugging purpose +#scp -i ~/.ssh/google_compute_engine \ +# -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ +# -r ${USERNAME}@${PUBLIC_IP}:/opt/stack/logs $ARTIFACTS/logs/devstack || true + +# If Boskos is being used then release the resource back to Boskos. +[ -z "${BOSKOS_HOST:-}" ] || python3 tests/scripts/boskos.py --release >> "$ARTIFACTS/logs/boskos.log" 2>&1 + +exit ${exit_code} diff --git a/tests/e2e/csi/cinder/cinder-testdriver.go b/tests/e2e/csi/cinder/cinder-testdriver.go index e0ef0e2a7f..45cfd49121 100644 --- a/tests/e2e/csi/cinder/cinder-testdriver.go +++ b/tests/e2e/csi/cinder/cinder-testdriver.go @@ -46,12 +46,12 @@ func initCinderDriver(name string, manifests ...string) storageframework.TestDri "topology.cinder.csi.openstack.org/zone", }, Capabilities: map[storageframework.Capability]bool{ - storageframework.CapPersistence: true, - storageframework.CapFsGroup: true, - storageframework.CapExec: true, - storageframework.CapMultiPODs: true, - storageframework.CapBlock: true, - storageframework.CapSnapshotDataSource: true, + storageframework.CapPersistence: true, + storageframework.CapFsGroup: true, + storageframework.CapExec: true, + storageframework.CapMultiPODs: true, + storageframework.CapBlock: true, + //storageframework.CapSnapshotDataSource: true, // storageframework.CapPVCDataSource: true, storageframework.CapTopology: true, }, diff --git a/tests/e2e/csi/cinder/csi-volumes.go b/tests/e2e/csi/cinder/csi-volumes.go index d9b5bd8d4f..f9abf7bd95 100644 --- a/tests/e2e/csi/cinder/csi-volumes.go +++ b/tests/e2e/csi/cinder/csi-volumes.go @@ -19,7 +19,7 @@ var CSITestSuites = []func() storageframework.TestSuite{ testsuites.InitVolumeModeTestSuite, testsuites.InitEphemeralTestSuite, //testsuites.InitVolumeIOTestSuite, - testsuites.InitSnapshottableTestSuite, + //testsuites.InitSnapshottableTestSuite, testsuites.InitMultiVolumeTestSuite, testsuites.InitFsGroupChangePolicyTestSuite, testsuites.InitTopologyTestSuite, diff --git a/tests/playbooks/roles/install-csi-cinder/README.md b/tests/playbooks/roles/install-csi-cinder/README.md new file mode 100644 index 0000000000..2fe2825aa7 --- /dev/null +++ b/tests/playbooks/roles/install-csi-cinder/README.md @@ -0,0 +1,10 @@ +The ansible role builds and uploads cinder-csi-plugin image and deploys in a k8s cluster. + +Prerequisites: + +* The playbook is running on a host with devstack installed. +* golang, docker and kubectl should be installed. +* docker registry is up and running. +* GOPATH should be configured in {{ global_env }} +* KUBECONFIG should be configured in {{ global_env }} +* k8s cluster is running inside VMs on the devstack host. diff --git a/tests/playbooks/roles/install-csi-cinder/defaults/main.yaml b/tests/playbooks/roles/install-csi-cinder/defaults/main.yaml new file mode 100644 index 0000000000..796335d79e --- /dev/null +++ b/tests/playbooks/roles/install-csi-cinder/defaults/main.yaml @@ -0,0 +1,10 @@ +--- +github_pr: 123 +devstack_workdir: "{{ ansible_user_dir }}/devstack" + +# Used for uploading image to local registry. +image_registry_host: localhost +# Used for access the private registry image from k8s +remote_registry_host: "{{ ansible_default_ipv4.address }}" + +k8s_log_dir: "/var/log/" diff --git a/tests/playbooks/roles/install-csi-cinder/tasks/main.yaml b/tests/playbooks/roles/install-csi-cinder/tasks/main.yaml new file mode 100644 index 0000000000..c52cea66ca --- /dev/null +++ b/tests/playbooks/roles/install-csi-cinder/tasks/main.yaml @@ -0,0 +1,210 @@ +--- +- name: Ensure CPO source folder + shell: + executable: /bin/bash + cmd: | + rm -rf $GOPATH/src/k8s.io/cloud-provider-openstack + mkdir -p $GOPATH/src/k8s.io; cd $_ + git clone https://github.com/kubernetes/cloud-provider-openstack + cd cloud-provider-openstack + git fetch origin +refs/pull/{{ github_pr }}/merge + git checkout FETCH_HEAD; git checkout -b PR{{ github_pr }} + +- name: Build and upload cinder-csi-plugin image + shell: + executable: /bin/bash + cmd: | + cd $GOPATH/src/k8s.io/cloud-provider-openstack + + REGISTRY={{ image_registry_host }} \ + VERSION={{ github_pr }} \ + IMAGE_NAMES=cinder-csi-plugin \ + make upload-image-amd64 + +- name: Prepare cloud config + shell: + executable: /bin/bash + cmd: | + kubectl -n kube-system get secret cloud-config >/dev/null 2>&1 + if [ $? -eq 0 ]; then + exit 0 + fi + + set -ex + + set +x; source {{ devstack_workdir }}/openrc admin admin > /dev/null; set -x + tenant_id=$(openstack project show demo -c id -f value) + set +x; source {{ devstack_workdir }}/openrc demo demo > /dev/null; set -x + cat < {{ ansible_user_dir }}/cloud.conf + [Global] + auth-url=${OS_AUTH_URL} + username=${OS_USERNAME} + password=${OS_PASSWORD} + region=${OS_REGION_NAME} + tenant-id=$tenant_id + domain-id=default + + [BlockStorage] + bs-version=v3 + EOF + + kubectl create secret -n kube-system generic cloud-config --from-file={{ ansible_user_dir }}/cloud.conf + +- name: Replace manifests + shell: + executable: /bin/bash + cmd: | + # replace manifest cloud secret file + + b64data=`cat {{ ansible_user_dir }}/cloud.conf | base64 -w 0` + cd $GOPATH/src/k8s.io/cloud-provider-openstack + + sed -i "/cloud\.conf/c\ cloud.conf: $b64data" manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml + + # replace image with built image + sed -i "s#docker.io/k8scloudprovider/cinder-csi-plugin:latest#{{ remote_registry_host }}/cinder-csi-plugin-amd64:{{ github_pr }}#" manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml + + sed -i "s#docker.io/k8scloudprovider/cinder-csi-plugin:latest#{{ remote_registry_host }}/cinder-csi-plugin-amd64:{{ github_pr }}#" manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml + +- name: Deploy openstack-cloud-controller-manager + shell: + executable: /bin/bash + cmd: | + set -x + + cat <