DM-47512: Upgrade MariaDB in Qserv to the latest stable version 11.4.4

admin/local/docker/compose/docker-compose.yml

@@ -103,7 +103,7 @@ services:
         << : *worker-xrootd
         command: >
           entrypoint worker-xrootd
-          --db-uri mysql://[email protected]:3306
+          --db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306
           --db-admin-uri mysql://root:[email protected]:3306
           --vnid-config "@/usr/local/lib64/libreplica.so {{db_uri}}/qservw_worker 0 0"
           --repl-instance-id qserv_proj
@@ -138,8 +138,8 @@ services:
         << : *worker-cmsd
         command: >
           entrypoint worker-cmsd
-          --db-uri mysql://qsmaster@worker_mariadb_0:3306
-          --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --db-uri mysql://qsmaster:CHANGEME2@worker_mariadb_0:3306
+          --vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
           --results-dirname /qserv/data/results
           --repl-instance-id qserv_proj
           --repl-auth-key replauthkey
@@ -208,9 +208,9 @@ services:
         << : *worker-xrootd
         command: >
           entrypoint --log-level DEBUG worker-xrootd
-          --db-uri mysql://[email protected]:3306?socket={{db_socket}}
+          --db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306?socket={{db_socket}}
           --db-admin-uri mysql://root:[email protected]:3306?socket={{db_socket}}
-          --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
           --repl-instance-id qserv_proj
           --repl-auth-key replauthkey
           --repl-admin-auth-key=repladminauthkey
@@ -244,8 +244,8 @@ services:
         << : *worker-cmsd
         command: >
           entrypoint --log-level DEBUG worker-cmsd
-          --db-uri mysql://qsmaster@worker_mariadb_1:3306?socket=/qserv/mariadb/run/mysqld.sock
-          --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --db-uri mysql://qsmaster:CHANGEME2@worker_mariadb_1:3306?socket=/qserv/mariadb/run/mysqld.sock
+          --vnid-config "@/usr/local/lib64/libreplica.so mysql://qsmaster:CHANGEME2@127.0.0.1:3306/qservw_worker 0 0"
           --results-dirname /qserv/data/results
           --repl-instance-id qserv_proj
           --repl-auth-key replauthkey
@@ -357,7 +357,7 @@ services:
         init: true
         command: >
           entrypoint --log-level DEBUG proxy
-          --db-uri mysql://[email protected]:3306?socket={{db_socket}}
+          --db-uri mysql://qsmaster:CHANGEME2@127.0.0.1:3306?socket={{db_socket}}
           --db-admin-uri mysql://root:[email protected]:3306?socket={{db_socket}}
           --targs db_socket=/qserv/mariadb/run/mysqld.sock
           --xrootd-manager czar_xrootd
@@ -395,7 +395,7 @@ services:
         init: true
         command: >
           entrypoint --log-level DEBUG czar-http
-          --db-uri mysql://qsmaster@czar_mariadb:3306/
+          --db-uri mysql://qsmaster:CHANGEME2@czar_mariadb:3306/
           --xrootd-manager czar_xrootd
           --czar-name http
           --http-port 4048

admin/tools/docker/base/Dockerfile

@@ -112,6 +112,7 @@ RUN cd /usr/share/java \
 RUN cd /tmp \
     && git clone https://github.com/lsst/mysqlproxy \
     && cd mysqlproxy \
+    && git checkout 0.8.5.lsst8 \
     && cp /usr/lib64/pkgconfig/lua-5.1.pc /usr/lib64/pkgconfig/lua5.1.pc \
     && ./configure \
     && make \

admin/tools/docker/mariadb/Dockerfile

@@ -15,7 +15,7 @@ FROM ubuntu:focal AS mariadb-scisql-builder
 
 RUN apt-get update \
     && apt-get install -y curl \
-    && curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-10.6.8" \
+    && curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash -s -- --mariadb-server-version="mariadb-11.4.4" \
     && apt-get install -y g++ make git python3 libmariadb-dev \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 0
 
@@ -40,7 +40,7 @@ RUN chmod a+r /tmp/scisql/build/deploy.mysql
 # off-the-shelf mariadb image.  This produces a generic "mariadb with scisql installed" container image.
 #-------------------------------------------------------------------------------------------------------------
 
-FROM mariadb:10.6.8 AS mariadb-scisql
+FROM mariadb:11.4.4 AS mariadb-scisql
 COPY --from=mariadb-scisql-builder /tmp/scisql/build/libscisql-scisql_0.3.so /usr/lib/mysql/plugin
 COPY --from=mariadb-scisql-builder /tmp/scisql/build/deploy.mysql /docker-entrypoint-initdb.d/scisql.sql
 

doc/admin/admin-upgrade.rst

@@ -0,0 +1,30 @@
+.. _admin-upgrade:
+
+Instructins for upgrading Qserv to newer releases
+=================================================
+
+.. _admin-upgrade-mariadb-11-4-4:
+
+Upgrading Qserv to MariaDB 11.4.4
+---------------------------------
+
+MariaDB 11.4.4 is a major release that introduces a number of new features and
+improvements. If your instance of Qserv is still based on MariaDB 10.6.8, you
+need to upgrade your data to the new version.
+
+To upgrade your Qserv installation to MariaDB 11.4.4, follow these steps:
+
+- Read the `MariaDB 11.4.4 release notes <https://mariadb.com/kb/en/mariadb-11-4-4-release-notes>`_ to learn about the new features and improvements.
+- Read the guide `Upgrading from MariaDB 10.6 to MariaDB 10.11 <https://mariadb.com/kb/en/upgrading-from-mariadb-10-6-to-mariadb-10-11>`_ to learn how to upgrade your databases.
+- Turn Qserv into the database-only mode by ensuring that all but the MariaDB services are being run.
+- Log into the MariaDB containers and run the database upgrade command as shown below:
+
+  .. code-block:: bash
+
+    mariadb-upgrade -P3306 -h127.0.0.1 --protocol=tcp -uroot -p******
+
+  Where ``-P`` specifies the port number, ``-h`` specifies the host, ``-u`` specifies the user, and ``-p`` specifies the password.
+  Note that the port number may be different in your installation.
+
+Experiments with the production databases at USDF (SLAC) have shown that the upgrade process is safe and reliable.
+However, it is still recommended to make a backup of your data before proceeding with the upgrade.

doc/admin/index.rst

@@ -5,10 +5,18 @@
 Administrator's Guide
 #####################
 
+.. warning::
+
+   This version of Qserv uses MariaDB version 11.4.4. If you still have an older release based on
+   MariaDB 10.6.8 then you should upgrade your databases as described in the following section:
+
+   - :ref:`admin-upgrade-mariadb-11-4-4`
+
 .. toctree::
    :maxdepth: 4
 
    k8s
    row-counters
    data-table-indexes
    director-index
+   admin-upgrade

itest_src/datasets/case01/data/schema/Filter.schema

@@ -1,39 +1,6 @@
--- MySQL dump 10.13  Distrib 5.1.61, for redhat-linux-gnu (x86_64)
---
--- Host: localhost    Database: case01_m
--- ------------------------------------------------------
--- Server version	5.1.61-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `Filter`
---
-
-DROP TABLE IF EXISTS `Filter`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `Filter` (
   `filterId` tinyint(4) NOT NULL,
   `filterName` char(3) NOT NULL,
   `photClam` float NOT NULL,
   `photBW` float NOT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-03-28 10:31:56

itest_src/datasets/case01/data/schema/LeapSeconds.schema

@@ -1,25 +1,3 @@
--- MySQL dump 10.13  Distrib 5.1.61, for redhat-linux-gnu (x86_64)
---
--- Host: localhost    Database: rplante_PT1_2_u_pt12prod_im3000
--- ------------------------------------------------------
--- Server version	5.1.61-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `LeapSeconds`
---
-
-DROP TABLE IF EXISTS `LeapSeconds`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `LeapSeconds` (
   `whenJd` float NOT NULL,
   `offset` float NOT NULL,
@@ -29,14 +7,3 @@ CREATE TABLE `LeapSeconds` (
   `whenUtc` bigint(20) DEFAULT NULL,
   `whenTai` bigint(20) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-04-16 10:56:43

itest_src/datasets/case01/data/schema/Logs.schema

@@ -1,25 +1,3 @@
--- MySQL dump 10.13  Distrib 5.5.20, for Linux (x86_64)
---
--- Host: localhost    Database: test
--- ------------------------------------------------------
--- Server version	5.5.20-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `Logs`
---
-
-DROP TABLE IF EXISTS `Logs`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `Logs` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `HOSTID` varchar(80) DEFAULT NULL,
@@ -48,14 +26,3 @@ CREATE TABLE `Logs` (
   PRIMARY KEY (`id`),
   KEY `a` (`RUNID`)
 ) ENGINE=MyISAM AUTO_INCREMENT=1001 DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-04-16 16:31:24

itest_src/datasets/case01/data/schema/Object.schema

@@ -1,25 +1,3 @@
--- MySQL dump 10.13  Distrib 5.1.61, for redhat-linux-gnu (x86_64)
---
--- Host: localhost    Database: case01_m
--- ------------------------------------------------------
--- Server version	5.1.61-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `Object`
---
-
-DROP TABLE IF EXISTS `Object`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `Object` (
   `objectId` bigint(20) NOT NULL,
   `iauId` char(34) DEFAULT NULL,
@@ -250,14 +228,3 @@ CREATE TABLE `Object` (
   `yFlags` int(11) DEFAULT NULL,
   `varBinaryField` varbinary(100) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-03-28 10:31:56

itest_src/datasets/case01/data/schema/RefSrcMatch.schema

@@ -1,25 +1,3 @@
--- MySQL dump 10.13  Distrib 5.1.61, for redhat-linux-gnu (x86_64)
---
--- Host: localhost    Database: tmp1
--- ------------------------------------------------------
--- Server version	5.1.61-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `RefSrcMatch`
---
-
-DROP TABLE IF EXISTS `RefSrcMatch`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `RefSrcMatch` (
   `refObjectId` bigint(20) DEFAULT NULL,
   `sourceId` bigint(20) DEFAULT NULL,
@@ -34,14 +12,3 @@ CREATE TABLE `RefSrcMatch` (
   KEY `sourceId` (`sourceId`),
   KEY `refObjectId` (`refObjectId`)
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-04-16 15:20:06

itest_src/datasets/case01/data/schema/Science_Ccd_Exposure.schema

@@ -1,25 +1,3 @@
--- MySQL dump 10.13  Distrib 5.1.61, for redhat-linux-gnu (x86_64)
---
--- Host: localhost    Database: case01_m
--- ------------------------------------------------------
--- Server version	5.1.61-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `Science_Ccd_Exposure`
---
-
-DROP TABLE IF EXISTS `Science_Ccd_Exposure`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `Science_Ccd_Exposure` (
   `scienceCcdExposureId` bigint(20) NOT NULL,
   `visit` int(11) NOT NULL,
@@ -67,14 +45,3 @@ CREATE TABLE `Science_Ccd_Exposure` (
   `poly` binary(120) NOT NULL,
   `flags` int(11) NOT NULL DEFAULT '0'
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2012-03-28 10:31:57