MDBF-814 - Add Columnstore upgrade test

Changes: - libvirt: - add minor | columnstore builder - suggest + tags for major/minor tests - suggest adding test_mode to builder name - bash library: - fix syntax for creating columnstore structures - move here: get_columnstore_logs() - deb upgrade: - the big comment block was deleted because dependency collection was handled in bash library - columnstore package list is now static. Mainly because it will pull CMAPI and was agreed not to install / upgrade it. - rpm upgrade: - columnstore test_mode option was missing - handled extra restart if test_mode is columnstore - remove chunks of commented code. Same reason as in deb upgrade. What will this patch do: - install the server and the CS plugin. Create structures (query) - upgrade both - verify structures (query) LDD and REQS comparison are OFF for CS.
fauust · Nov 4, 2024 · 10fb44b · 10fb44b
1 parent 3b00708
commit 10fb44b
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 235 deletions.
diff --git a/constants.py b/constants.py
@@ -256,7 +256,10 @@
         # Currently there are no VMs for x86 and s390x and OpenSUSE and SLES
         if arch not in ["s390x", "x86"] and "sles" not in os_i:
             builders_install.append(builder_name_autobake + "-install")
-            builders_upgrade.append(builder_name_autobake + "-minor-upgrade")
+            builders_upgrade.append(builder_name_autobake + "-minor-upgrade-all")
+            builders_upgrade.append(
+                builder_name_autobake + "-minor-upgrade-columnstore"
+            )
             builders_upgrade.append(builder_name_autobake + "-major-upgrade")
 
 builders_galera = list(

diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg
@@ -338,7 +338,7 @@ for builder_name in builders_install:
         util.BuilderConfig(
             name=major_upgrade_name,
             workernames=libvirt_worker_name,
-            tags=[os_name, builder_type, "upgrade", "kvm"],
+            tags=[os_name, builder_type, "upgrade", "kvm", "major", "server"],
             collapseRequests=True,
             nextBuild=nextBuild,
             canStartBuild=canStartBuild,
@@ -357,14 +357,14 @@ for builder_name in builders_install:
         )
     )
 
-    # Add minor upgrade builder
+    # Add minor upgrade builder, test_mode=all
     minor_upgrade_name = "-".join(builder_name.split("-")[:5]) + "-minor-upgrade"
 
     c["builders"].append(
         util.BuilderConfig(
-            name=minor_upgrade_name,
+            name=minor_upgrade_name + "-all",
             workernames=libvirt_worker_name,
-            tags=[os_name, builder_type, "upgrade", "kvm"],
+            tags=[os_name, builder_type, "upgrade", "kvm", "minor", "all"],
             collapseRequests=True,
             nextBuild=nextBuild,
             canStartBuild=canStartBuild,
@@ -383,6 +383,30 @@ for builder_name in builders_install:
         )
     )
 
+    # Add minor upgrade builder, test_mode=columnstore
+    c["builders"].append(
+        util.BuilderConfig(
+            name=minor_upgrade_name + "-columnstore",
+            workernames=libvirt_worker_name,
+            tags=[os_name, builder_type, "upgrade", "kvm", "minor", "columnstore"],
+            collapseRequests=True,
+            nextBuild=nextBuild,
+            canStartBuild=canStartBuild,
+            properties={
+                "systemdCapability": "yes",
+                "needsGalera": "no",
+                "dist_name": os_name,
+                "version_name": os_info[os_info_name]["version_name"],
+                "arch": build_arch,
+                "test_mode": "columnstore",
+                "test_type": "minor",
+                "BB_CI": True,
+                "artifactsURL": artifactsURL,
+            },
+            factory=factory_upgrade,
+        )
+    )
+
 c["logEncoding"] = "utf-8"
 
 c["multiMaster"] = True

diff --git a/scripts/bash_lib.sh b/scripts/bash_lib.sh
@@ -437,6 +437,24 @@ upgrade_test_type() {
   esac
 }
 
+get_columnstore_logs() {
+  if [[ $test_mode == "columnstore" ]]; then
+    bb_log_info "storing Columnstore logs in columnstore_logs"
+    set +ex
+    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
+    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
+    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
+      f=/var/log/mariadb/columnstore/$f
+      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
+      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
+    done
+    for f in /tmp/columnstore_tmp_files/*; do
+      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
+      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
+    done
+  fi
+}
+
 check_mariadb_server_and_create_structures() {
   # All the commands below should succeed
   set -e
@@ -450,7 +468,7 @@ check_mariadb_server_and_create_structures() {
   sudo mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
   sudo mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
   if [[ $test_mode == "columnstore" ]]; then
-    if ! mysql -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
+    if ! sudo mariadb -e "CREATE TABLE db.t_columnstore(a INT, c VARCHAR(8)) ENGINE=ColumnStore; SHOW CREATE TABLE db.t_columnstore; INSERT INTO db.t_columnstore VALUES (1,'foo'),(2,'bar')"; then
       get_columnstore_logs
       exit 1
     fi

diff --git a/scripts/deb-upgrade.sh b/scripts/deb-upgrade.sh
@@ -75,7 +75,7 @@ case $test_mode in
       bb_log_warn "Columnstore isn't necessarily built on Sid, the test will be skipped"
       exit
     fi
-    package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | sort -u | xargs)
+    package_list="mariadb-server mariadb-plugin-columnstore"
     ;;
   *)
     bb_log_err "unknown test mode: $test_mode"
@@ -98,25 +98,6 @@ sudo sh -c "echo 'Pin-Priority: 1000' >> /etc/apt/preferences.d/release"
 # apt get update may be running in the background (Ubuntu start).
 apt_get_update
 
-# //TEMP this is called from bash_lib, not good.
-get_columnstore_logs() {
-  if [[ $test_mode == "columnstore" ]]; then
-    bb_log_info "storing Columnstore logs in columnstore_logs"
-    set +ex
-    # It is done in such a weird way, because Columnstore currently makes its logs hard to read
-    # //TEMP this is fragile and weird (test that /var/log/mariadb/columnstore exist)
-    for f in $(sudo ls /var/log/mariadb/columnstore | xargs); do
-      f=/var/log/mariadb/columnstore/$f
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" 1>>/home/buildbot/columnstore_logs 2>&1
-    done
-    for f in /tmp/columnstore_tmp_files/*; do
-      echo "----------- $f -----------" >>/home/buildbot/columnstore_logs
-      sudo cat "$f" | sudo tee -a /home/buildbot/columnstore_logs 2>&1
-    done
-  fi
-}
-
 # Install previous release
 # Debian installation/upgrade/startup always attempts to execute mysql_upgrade, and
 # also run mysqlcheck and such. Due to MDEV-14622, they are subject to race condition,
@@ -230,72 +211,6 @@ check_mariadb_server_and_verify_structures
 collect_dependencies new deb
 store_mariadb_server_info new
 
-# //TEMP what needs to be done here?
-# # Dependency information for new binaries/libraries
-# set +x
-# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(dpkg-query -L $(dpkg -l | grep mariadb | awk '{print $2}' | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort -u | xargs); do
-#   echo "=== $i"
-#   ldd $i | sort | sed 's/(.*)//'
-# done >/home/buildbot/ldd.new
-# set -x
-# case "$systemdCapability" in
-#   yes)
-#     ls -l /lib/systemd/system/mariadb.service
-#     ls -l /etc/systemd/system/mariadb.service.d/migrated-from-my.cnf-settings.conf
-#     ls -l /etc/init.d/mysql || true
-#     systemctl --no-pager status mariadb.service
-#     systemctl --no-pager status mariadb
-#     systemctl --no-pager status mysql
-#     systemctl --no-pager status mysqld
-#     systemctl --no-pager is-enabled mariadb
-#     ;;
-#   no)
-#     bb_log_info "Steps related to systemd will be skipped"
-#     ;;
-#   *)
-#     bb_log_err "It should never happen, check your configuration (systemdCapability property is not set or is set to a wrong value)"
-#     exit 1
-#     ;;
-# esac
-# set +e
-# # This output is for informational purposes
-# diff -u /tmp/engines.old /tmp/engines.new
-# diff -u /tmp/plugins.old /tmp/plugins.new
-# case "$branch" in
-#   "$development_branch")
-#     bb_log_info "Until $development_branch is GA, the list of plugins/engines might be unstable, skipping the check"
-#     ;;
-#   *)
-#     # Only fail if there are any disappeared/changed engines or plugins
-#     disappeared_or_changed=$(comm -23 /tmp/engines.old /tmp/engines.new | wc -l)
-#     if ((disappeared_or_changed != 0)); then
-#       bb_log_err "the lists of engines in the old and new installations differ"
-#       exit 1
-#     fi
-#     if [[ $test_type == "minor" ]]; then
-#       disappeared_or_changed=$(comm -23 /tmp/plugins.old /tmp/plugins.new | wc -l)
-#       if ((disappeared_or_changed != 0)); then
-#         bb_log_err "the lists of plugins in the old and new installations differ"
-#         exit 1
-#       fi
-#     fi
-#     set -o pipefail
-#     if [[ $test_mode == "all" ]]; then
-#       set -o pipefail
-#       if wget -q --timeout=20 --no-check-certificate "https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot/baselines/ldd.${major_version}.${version_name}.$(deb_arch)" -O /tmp/ldd.baseline; then
-#         ldd_baseline=/tmp/ldd.baseline
-#       else
-#         ldd_baseline=/buildbot/ldd.old
-#       fi
-#       if ! diff -U1000 $ldd_baseline /home/buildbot/ldd.new | (grep -E '^[-+]|^ =' || true); then
-#         bb_log_err "something has changed in the dependencies of binaries or libraries. See the diff above"
-#         exit 1
-#       fi
-#     fi
-#     set +o pipefail
-#     ;;
-# esac
-
 check_upgraded_versions
 
 bb_log_ok "all done"
diff --git a/scripts/rpm-upgrade.sh b/scripts/rpm-upgrade.sh
@@ -52,46 +52,20 @@ case $test_mode in
   server)
     package_list="MariaDB-server MariaDB-client"
     ;;
+  columnstore)
+    package_list=$(rpm_repoquery)
+    if ! echo "$package_list" | grep -q columnstore-engine; then
+      bb_log_warn "Columnstore was not found in the released packages, the test will not be run"
+      exit
+    fi
+    package_list="MariaDB-server MariaDB-columnstore-engine"
+    ;;
   *)
     bb_log_err "unknown test mode ($test_mode)"
     exit 1
     ;;
 esac
 
-# case $test_mode in
-#   all | deps | columnstore)
-#     if [[ $test_mode == "all" ]]; then
-#       if echo "$package_list" | grep -qi columnstore; then
-#         bb_log_warn "due to MCOL-4120 and other issues, Columnstore upgrade will be tested separately"
-#       fi
-#       package_list=$(echo "$package_list" | grep MariaDB |
-#         grep -viE 'galera|columnstore' | sed -e 's/<name>//' |
-#         sed -e 's/<\/name>//' | sort -u | xargs)
-#     elif [[ $test_mode == "deps" ]]; then
-#       package_list=$(echo "$package_list" |
-#         grep -iE 'MariaDB-server|MariaDB-test|MariaDB-client|MariaDB-common|MariaDB-compat' |
-#         sed -e 's/<name>//' | sed -e 's/<\/name>//' | sort -u | xargs)
-#     elif [[ $test_mode == "columnstore" ]]; then
-#       if ! echo "$package_list" | grep -q columnstore; then
-#         bb_log_warn "columnstore was not found in the released packages, the test will not be run"
-#         exit
-#       fi
-#       package_list="MariaDB-server MariaDB-columnstore-engine"
-#     fi
-
-#     if [[ $arch == ppc* ]]; then
-#       package_list=$(echo "$package_list" | xargs -n1 | sed -e 's/MariaDB-compat//gi' | xargs)
-#     fi
-#     ;;
-#   server)
-#     package_list="MariaDB-server MariaDB-client"
-#     ;;
-#   *)
-#     bb_log_err "unknown test mode: $test_mode"
-#     exit 1
-#     ;;
-# esac
-
 bb_log_info "Package_list: $package_list"
 
 # # //TEMP this needs to be implemented once we have SLES VM in new BB
@@ -214,46 +188,6 @@ if [[ $package_version == "$old_version" ]]; then
   exit
 fi
 
-# # Store dependency information for old binaries/libraries:
-# # - names starting with "mysql*" in the directory where mysqld is located;
-# # - names starting with "mysql*" in the directory where mysql is located;
-# # - everything in the plugin directories installed by any MariaDB packages
-# set +x
-# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(rpm -ql $(rpm -qa | grep MariaDB | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort | uniq | xargs); do
-#   echo "=== $i"
-#   ldd $i | sort | sed 's/(.*)//'
-# done >/home/buildbot/ldd.old
-# set -x
-
-# # Prepare yum/zypper configuration for installation of the new packages
-# # //TEMP again not sure this is needed
-# set -e
-# if [[ $test_type == "major" ]]; then
-#   bb_log_info "remove old packages for major upgrade"
-#   packages_to_remove=$(rpm -qa | grep 'MariaDB-' | awk -F'-' '{print $1"-"$2}' | xargs)
-#   sudo sh -c "$remove_command $packages_to_remove"
-#   rpm -qa | grep -iE 'maria|mysql' || true
-# fi
-# if [[ $test_mode == "deps" ]]; then
-#   sudo mv "$repo_location/MariaDB.repo" /tmp
-#   sudo rm -rf "$repo_location/*"
-#   sudo mv /tmp/MariaDB.repo "$repo_location/"
-#   sudo sh -c "$cleanup_command"
-# fi
-
-# Install the new packages:
-# Between 10.3 and 10.4(.2), required galera version changed from galera(-3) to galera-4.
-# It means that there will be no galera-4 in the "old" repo, and it's not among the local RPMs.
-# So, we need to add a repo for it
-# //TEMP this needs to be fixed
-# if [[ $test_type == "major" ]] && ((${major_version/10./} >= 3)) && ((${prev_major_version/10./} <= 4)); then
-#   sudo sh -c "echo '[galera]
-# name=Galera
-# baseurl=https://yum.mariadb.org/galera/repo4/rpm/$arch
-# gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
-# gpgcheck=1' > $repo_location/galera.repo"
-# fi
-
 # //TEMP upgrade does not work without this but why? Can't we fix it?
 if [[ $test_type == "major" ]]; then
   bb_log_info "remove old packages for major upgrade"
@@ -291,7 +225,7 @@ fi
 
 # Optionally (re)start the server
 set -e
-if [[ $test_type == "major" ]]; then
+if [[ $test_type == "major" ]] || [[ $test_mode == "columnstore" ]]; then
   control_mariadb_server restart
 fi
 
@@ -314,74 +248,6 @@ check_mariadb_server_and_verify_structures
 collect_dependencies new rpm
 store_mariadb_server_info new
 
-# # Dependency information for new binaries/libraries
-# set +x
-# for i in $(sudo which mysqld | sed -e 's/mysqld$/mysql\*/') $(which mysql | sed -e 's/mysql$/mysql\*/') $(rpm -ql $(rpm -qa | grep MariaDB | xargs) | grep -v 'mysql-test' | grep -v '/debug/' | grep '/plugin/' | sed -e 's/[^\/]*$/\*/' | sort | uniq | xargs); do
-#   echo "=== $i"
-#   ldd "$i" | sort | sed 's/(.*)//'
-# done >/home/buildbot/ldd.new
-# set -x
-# case "$systemdCapability" in
-#   yes)
-#     ls -l /usr/lib/systemd/system/mariadb.service
-#     ls -l /etc/systemd/system/mariadb.service.d/migrated-from-my.cnf-settings.conf
-#     ls -l /etc/init.d/mysql || true
-#     systemctl status mariadb.service --no-pager
-#     systemctl status mariadb --no-pager
-#     # Not done for SUSE due to MDEV-23044
-#     if [[ "$distro" != *"sles"* ]] && [[ "$distro" != *"suse"* ]]; then
-#       # Major upgrade for RPMs is remove / install, so previous configuration
-#       # could well be lost
-#       if [[ "$test_type" == "major" ]]; then
-#         sudo systemctl enable mariadb
-#       fi
-#       systemctl is-enabled mariadb
-#       systemctl status mysql --no-pager
-#       systemctl status mysqld --no-pager
-#     fi
-#     ;;
-#   no)
-#     bb_log_info "Steps related to systemd will be skipped"
-#     ;;
-#   *)
-#     bb_log_err "It should never happen, check your configuration (systemdCapability property is not set or is set to a wrong value)"
-#     exit 1
-#     ;;
-# esac
-# set +e
-
-# # Until $development_branch is GA, the list of plugins/engines might be unstable, skipping the check
-# # For major upgrade, no point to do the check at all
-# if [[ $major_version != "$development_branch" ]] && [[ $test_type != "major" ]]; then
-#   # This output is for informational purposes
-#   diff -u /tmp/engines.old /tmp/engines.new
-#   diff -u /tmp/plugins.old /tmp/plugins.new
-#   # Only fail if there are any disappeared/changed engines or plugins
-#   disappeared_or_changed=$(comm -23 /tmp/engines.old /tmp/engines.new | wc -l)
-#   if ((disappeared_or_changed != 0)); then
-#     bb_log_err "the lists of engines in the old and new installations differ"
-#     exit 1
-#   fi
-#   disappeared_or_changed=$(comm -23 /tmp/plugins.old /tmp/plugins.new | wc -l)
-#   if ((disappeared_or_changed != 0)); then
-#     bb_log_err "the lists of available plugins in the old and new installations differ"
-#     exit 1
-#   fi
-#   if [[ $test_mode == "all" ]]; then
-#     set -o pipefail
-#     if wget -q --timeout=20 --no-check-certificate "https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot/baselines/ldd.${major_version}.${distro}.${arch}" -O /tmp/ldd.baseline; then
-#       ldd_baseline=/tmp/ldd.baseline
-#     else
-#       ldd_baseline=/home/buildbot/ldd.old
-#     fi
-#     if ! diff -U1000 $ldd_baseline /home/buildbot/ldd.new | (grep -E '^[-+]|^ =' || true); then
-#       bb_log_err "something has changed in the dependencies of binaries or libraries. See the diff above"
-#       exit 1
-#     fi
-#   fi
-#   set +o pipefail
-# fi
-
 check_upgraded_versions
 
 bb_log_ok "all done"