Extended Controller's REST API to report database activities

Added the REST services to report database activities at Czar and
Qserv workers. Incremented the version number of the REST API.

src/admin/python/lsst/qserv/admin/replicationInterface.py

@@ -201,7 +201,7 @@ def __init__(
         self.repl_ctrl = urlparse(repl_ctrl_uri)
         self.auth_key = auth_key
         self.admin_auth_key = admin_auth_key
-        self.repl_api_version = 23
+        self.repl_api_version = 24
         _log.debug(f"ReplicationInterface %s", self.repl_ctrl)
 
     def version(self) -> str:

src/replica/HttpMetaModule.cc

@@ -34,7 +34,7 @@ using json = nlohmann::json;
 
 namespace lsst::qserv::replica {
 
-unsigned int const HttpMetaModule::version = 23;
+unsigned int const HttpMetaModule::version = 24;
 
 void HttpMetaModule::process(ServiceProvider::Ptr const& serviceProvider, string const& context,
                              qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp,

src/replica/HttpProcessor.cc

@@ -238,6 +238,18 @@ void HttpProcessor::registerServices() {
                                  HttpQservMonitorModule::process(self->controller(), self->name(),
                                                                  self->_processorConfig, req, resp, "WORKER");
                              });
+    httpServer()->addHandler("GET", "/replication/qserv/worker/db/:worker",
+                             [self](qhttp::Request::Ptr const req, qhttp::Response::Ptr const resp) {
+                                 HttpQservMonitorModule::process(self->controller(), self->name(),
+                                                                 self->_processorConfig, req, resp,
+                                                                 "WORKER-DB");
+                             });
+    httpServer()->addHandler("GET", "/replication/qserv/master/db",
+                             [self](qhttp::Request::Ptr const req, qhttp::Response::Ptr const resp) {
+                                 HttpQservMonitorModule::process(self->controller(), self->name(),
+                                                                 self->_processorConfig, req, resp,
+                                                                 "CZAR-DB");
+                             });
     httpServer()->addHandler("GET", "/replication/qserv/master/query",
                              [self](qhttp::Request::Ptr const req, qhttp::Response::Ptr const resp) {
                                  HttpQservMonitorModule::process(self->controller(), self->name(),

src/replica/HttpQservMonitorModule.cc

@@ -33,6 +33,9 @@
 // Qserv headers
 #include "css/CssAccess.h"
 #include "css/CssError.h"
+#include "replica/DatabaseMySQL.h"
+#include "replica/DatabaseMySQLTypes.h"
+#include "replica/DatabaseMySQLUtils.h"
 #include "global/intTypes.h"
 #include "replica/Common.h"
 #include "replica/Configuration.h"
@@ -139,6 +142,10 @@ json HttpQservMonitorModule::executeImpl(string const& subModuleName) {
         return _workers();
     else if (subModuleName == "WORKER")
         return _worker();
+    else if (subModuleName == "WORKER-DB")
+        return _workerDb();
+    else if (subModuleName == "CZAR-DB")
+        return _czarDb();
     else if (subModuleName == "QUERIES")
         return _userQueries();
     else if (subModuleName == "QUERY")
@@ -217,6 +224,44 @@ json HttpQservMonitorModule::_worker() {
     return result;
 }
 
+json HttpQservMonitorModule::_workerDb() {
+    debug(__func__);
+    checkApiVersion(__func__, 24);
+
+    auto const worker = params().at("worker");
+    unsigned int const timeoutSec = query().optionalUInt("timeout_sec", workerResponseTimeoutSec());
+
+    debug(__func__, "worker=" + worker);
+    debug(__func__, "timeout_sec=" + to_string(timeoutSec));
+
+    string const noParentJobId;
+    GetDbStatusQservMgtRequest::CallbackType const onFinish = nullptr;
+
+    auto const request = controller()->serviceProvider()->qservMgtServices()->databaseStatus(
+            worker, noParentJobId, onFinish, timeoutSec);
+    request->wait();
+
+    if (request->extendedState() != QservMgtRequest::ExtendedState::SUCCESS) {
+        string const msg = "database operation failed, error: " +
+                           QservMgtRequest::state2string(request->extendedState());
+        throw HttpError(__func__, msg);
+    }
+    json result = json::object();
+    result["status"] = request->info();
+    return result;
+}
+
+json HttpQservMonitorModule::_czarDb() {
+    debug(__func__);
+    checkApiVersion(__func__, 24);
+
+    // Connect to the master database. Manage the new connection via the RAII-style
+    // handler to ensure the transaction is automatically rolled-back in case of exceptions.
+    ConnectionHandler const h(Connection::open(Configuration::qservCzarDbParams("qservMeta")));
+    bool const full = true;
+    return json::object({{"status", database::mysql::processList(h.conn, full)}});
+}
+
 wbase::TaskSelector HttpQservMonitorModule::_translateTaskSelector(string const& func) const {
     wbase::TaskSelector selector;
     selector.includeTasks = query().optionalUInt("include_tasks", 0) != 0;

src/replica/HttpQservMonitorModule.h

@@ -56,6 +56,8 @@ class HttpQservMonitorModule : public HttpModule {
      *
      *   WORKERS    get the status info of many workers (possible selected by various criteria)
      *   WORKER     get the status info of a specific worker
+     *   WORKER-DB  get the database status of a specific worker
+     *   CZAR-DB    get the database status of Czar
      *   QUERIES    get user query info (queries selected by various criteria)
      *   QUERY      get user query info for a specific query
      *   CSS        get CSS configurations (the shared scan settings, etc.)
@@ -94,6 +96,18 @@ class HttpQservMonitorModule : public HttpModule {
      */
     nlohmann::json _worker();
 
+    /**
+     * Process a request for extracting various status info on the database
+     * service for select Qserv worker.
+     */
+    nlohmann::json _workerDb();
+
+    /**
+     * Process a request for extracting various status info on the database
+     * service of Czar.
+     */
+    nlohmann::json _czarDb();
+
     /**
      * Process a request for extracting a status on select user queries
      * launched at Qserv.