Add support for the file-based result delivery to worker's CMSD config

Added the corresponding configuration option to the entry-point CLI
for ingecting a value of the result folder into the config file of
of the worker CMSD. The configuration file has been extended as well
to allow serving files from workers via the XROOTD file-based protocol.
The SSI provider class has been extened to recognise result files
at the local filesystems of workers as valid XROOTD resources.

admin/local/docker/compose/docker-compose.yml

@@ -42,11 +42,13 @@ x-worker-repl:
 
 volumes:
     worker_0_data:
+    worker_0_results:
     worker_0_repl_cfg:
     worker_0_xrootd:
     worker_0_home:
     worker_0_mariadb_lib:
     worker_1_data:
+    worker_1_results:
     worker_1_repl_cfg:
     worker_1_xrootd:
     worker_1_home:
@@ -79,6 +81,9 @@ services:
             - type: volume
               source: worker_0_data
               target: /qserv/data
+            - type: volume
+              source: worker_0_results
+              target: /qserv/data/results
             - type: volume
               source: worker_0_mariadb_lib
               target: /var/lib/mysql
@@ -90,11 +95,15 @@ services:
           --db-uri mysql://[email protected]:3306
           --db-admin-uri mysql://root:[email protected]:3306
           --vnid-config "@/usr/local/lib64/libreplica.so {{db_uri}}/qservw_worker 0 0"
+          --results-dirname /qserv/data/results
           --cmsd-manager-name manager-xrootd
           --cmsd-manager-count 1
           --mysql-monitor-password CHANGEME_MONITOR
           --log-cfg-file=/config-etc/log/log-worker-xrootd.cnf
         volumes:
+            - type: volume
+              source: worker_0_results
+              target: /qserv/data/results
             - type: volume
               source: worker_0_xrootd
               target: /var/run/xrootd
@@ -113,10 +122,14 @@ services:
           entrypoint worker-cmsd
           --db-uri mysql://qsmaster@worker-db-0:3306
           --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --results-dirname /qserv/data/results
           --cmsd-manager-name manager-xrootd
           --cmsd-manager-count 1
         network_mode: "service:worker-xrootd-0"
         volumes:
+            - type: volume
+              source: worker_0_results
+              target: /qserv/data/results
             - type: volume
               source: worker_0_xrootd
               target: /var/run/xrootd
@@ -156,6 +169,9 @@ services:
             - type: volume
               source: worker_1_data
               target: /qserv/data
+            - type: volume
+              source: worker_1_results
+              target: /qserv/data/results
             - type: volume
               source: worker_1_mariadb_lib
               target: /var/lib/mysql
@@ -170,12 +186,16 @@ services:
           --db-uri mysql://[email protected]:3306?socket={{db_socket}}
           --db-admin-uri mysql://root:[email protected]:3306?socket={{db_socket}}
           --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --results-dirname /qserv/data/results
           --cmsd-manager-name manager-xrootd
           --cmsd-manager-count 1
           --mysql-monitor-password CHANGEME_MONITOR
           --targs db_socket=/qserv/mariadb/run/mysqld.sock
           --log-cfg-file=/config-etc/log/log-worker-xrootd.cnf
         volumes:
+            - type: volume
+              source: worker_1_results
+              target: /qserv/data/results
             - type: volume
               source: worker_1_xrootd
               target: /var/run/xrootd
@@ -197,10 +217,14 @@ services:
           entrypoint --log-level DEBUG worker-cmsd
           --db-uri mysql://qsmaster@worker-db-1:3306?socket=/qserv/mariadb/run/mysqld.sock
           --vnid-config "@/usr/local/lib64/libreplica.so mysql://[email protected]:3306/qservw_worker 0 0"
+          --results-dirname /qserv/data/results
           --cmsd-manager-name manager-xrootd
           --cmsd-manager-count 1
         network_mode: "service:worker-xrootd-1"
         volumes:
+            - type: volume
+              source: worker_1_results
+              target: /qserv/data/results
             - type: volume
               source: worker_1_xrootd
               target: /var/run/xrootd

admin/tools/docker/mariadb/Dockerfile

@@ -55,7 +55,7 @@ FROM mariadb-scisql AS lite-mariadb
 
 RUN useradd --uid 1000 qserv
 
-RUN mkdir -p /qserv/data /config-etc /var/log/mysql \
-    && chown qserv:qserv /qserv/data /config-etc /var/log/mysql
+RUN mkdir -p /qserv/data /qserv/data/results /config-etc /var/log/mysql \
+    && chown qserv:qserv /qserv/data /qserv/data/results /config-etc /var/log/mysql
 
 USER qserv

src/admin/python/lsst/qserv/admin/cli/entrypoint.py

@@ -55,6 +55,7 @@
     repl_admin_auth_key_option,
     repl_auth_key_option,
     repl_connection_option,
+    results_dirname_option,
     run_option,
     run_tests_option,
     targs_options,
@@ -551,6 +552,7 @@ def xrootd_manager(ctx: click.Context, **kwargs: Any) -> None:
 @pass_context
 @db_uri_option(help=worker_db_help)
 @vnid_config_option(required=True)
+@results_dirname_option()
 @cmsd_manager_name_option()
 @cmsd_manager_count_option()
 @debug_option()
@@ -584,6 +586,7 @@ def worker_cmsd(ctx: click.Context, **kwargs: Any) -> None:
 @db_uri_option(help=worker_db_help)
 @db_admin_uri_option(help=admin_worker_db_help)
 @vnid_config_option(required=True)
+@results_dirname_option()
 @cmsd_manager_name_option()
 @cmsd_manager_count_option()
 @mysql_monitor_password_option()
@@ -605,6 +608,7 @@ def worker_xrootd(ctx: click.Context, **kwargs: Any) -> None:
         db_uri=targs["db_uri"],
         db_admin_uri=targs["db_admin_uri"],
         vnid_config=targs["vnid_config"],
+        results_dirname=targs["results_dirname"],
         mysql_monitor_password=targs["mysql_monitor_password"],
         db_qserv_user=targs["db_qserv_user"],
         cmsd_worker_cfg_file=targs["cmsd_worker_cfg_file"],

src/admin/python/lsst/qserv/admin/cli/options.py

@@ -146,6 +146,14 @@ def __call__(self, f: Callable) -> Callable:
     " source (static string, a file or worker database)."
 )
 
+results_dirname_option = partial(
+    click.option,
+    "--results-dirname",
+    help="Path to a folder where worker stores result sets of queries.",
+    default="/qserv/data/results",
+    show_default=True,
+)
+
 
 xrootd_manager_option = partial(
     click.option,

src/admin/python/lsst/qserv/admin/cli/script.py

@@ -412,6 +412,7 @@ def enter_worker_xrootd(
     db_uri: str,
     db_admin_uri: str,
     vnid_config: str,
+    results_dirname: str,
     mysql_monitor_password: str,
     db_qserv_user: str,
     cmsd_worker_cfg_file: str,
@@ -436,6 +437,8 @@ def enter_worker_xrootd(
     vnid_config : str
         The config parameters used by the qserv cmsd to get the vnid
         from the specified source (static string, a file or worker database).
+    results_dirname : str
+        A path to a folder where query results will be stored.
     mysql_monitor_password : str
         The password used by applications that monitor via the worker database.
     db_qserv_user : str

src/admin/templates/xrootd/etc/cmsd-worker.cf.jinja

@@ -4,14 +4,18 @@ all.role server
 cms.vnid {{ vnid_config }}
 
 # Use XrdSsi plugin
-xrootd.fslib -2 libXrdSsi.so
+xrootd.fslib -2 libXrdSsi.so default
 ssi.svclib libxrdsvc.so
 oss.statlib -2 -arevents libXrdSsi.so
 
 # Force disable asyncronous access
 # because of XrdSsi
 xrootd.async off
 
+# Access to the files at the file system as specified by the "default" option
+# in the statement "xrootd.fslib" above.
+ssi.fspath {{ results_dirname }}
+
 ssi.trace all debug
 
 ########################################
@@ -25,6 +29,7 @@ all.adminpath /var/run/xrootd
 # "nolock" directive prevents write-locking and is important for qserv
 # qserv is hardcoded for these paths.
 all.export / nolock
+all.export {{ results_dirname }}
 
 # Specify that no significant free space is required on servers
 # Indeed current configuration doesn't expect to be dynamically

src/admin/templates/xrootd/etc/xrdssi.cf.jinja

@@ -83,3 +83,7 @@ maxtransmits = 50
 # If more than this number of large transmits is happening at once, wait to
 # start more transmits until some are done.
 maxalreadytransmitting = 10
+
+[results]
+# The name of a folder where query results will be stored.
+dirname = {{ results_dirname }}

src/xrdsvc/CMakeLists.txt

@@ -14,6 +14,8 @@ target_include_directories(qserv_xrdsvc PRIVATE
 )
 
 target_link_libraries(qserv_xrdsvc PUBLIC
+    boost_filesystem
+    boost_system
     log
     XrdSsiLib
 )

src/xrdsvc/SsiProvider.cc

@@ -32,6 +32,7 @@
 #include <sys/types.h>
 
 // Third party headers
+#include "boost/filesystem.hpp"
 #include "XrdSsi/XrdSsiCluster.hh"
 #include "XrdSsi/XrdSsiLogger.hh"
 
@@ -176,6 +177,16 @@ XrdSsiProvider::rStat SsiProviderServer::QueryResource(char const* rName, char c
         return isPresent;
     }
 
+    // Treat other resources as absolute path names of files
+    boost::filesystem::path const path(rName);
+    if (path.is_absolute()) {
+        boost::system::error_code ec;
+        if (boost::filesystem::exists(path, ec) && !ec.value()) {
+            LOGS(_log, LOG_LVL_DEBUG, "SsiProvider File Resource " << rName << " recognized");
+            return isPresent;
+        }
+    }
+
     LOGS(_log, LOG_LVL_DEBUG, "SsiProvider Query " << rName << " invalid");
     return notPresent;
 }