From dc7da3532f02bab93741530cd51eeb8c6654db05 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 31 May 2023 16:40:45 +0000 Subject: [PATCH 1/8] Adding container config from #29 --- .gitignore | 49 +++++++ config/executor_manager.json | 63 ++++++++- server/executor_manager/executor_process.cpp | 129 ++++++++++--------- server/executor_manager/settings.cpp | 89 +++++++++++++ server/executor_manager/settings.hpp | 118 ++++++++++++++++- 5 files changed, 384 insertions(+), 64 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8364c9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +bin/ +configuration/ +volumes/ +containers/config/htpasswd diff --git a/config/executor_manager.json b/config/executor_manager.json index 6e4cf8b..4695330 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -1,16 +1,73 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10000, + "rdma_device_port": 10004, "resource_manager_address": "", "resource_manager_port": 0, "resource_manager_secret": 0 }, "executor": { - "use_docker": false, + "sandbox_type": "process", "repetitions": 100, "warmup_iters": 0, "pin_threads": false - } + }, + "sandbox-configuration": [ + { + "key": "sarus", + "value": { + "index": 1, + "data": { + "user": "mcopik", + "name": "spcleth/hpc-disagg:rfaas-executor-daint", + "devices": [ + "/dev/kgni0", "dev/kdreg" + ], + "mounts": [ + "/opt/cray", + "/tmp/drcc.sock", + "/etc/opt/cray/rdma-credentials", + "/etc/alternatives/cray-ugni", + "/etc/alternatives/cray-xpmem", + "/etc/alternatives/cray-alps", + "/etc/alternatives/cray-udreg", + "/etc/alternatives/cray-wlm_detect" + ], + "mount_filesystem": [ + "/scratch/snx3000/{user}" + ], + "env": [ + { + "key": "LD_LIBRARY_PATH", + "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/" + } + ] + } + } + }, + { + "key": "docker", + "value": { + "index": 0, + "data": { + "image": "rfaas-registry/rfaas-base", + "network": "mynet", + "ip": "172.31.82.202", + "volume": "/home/ubuntu/rfaas/containers/opt", + "registry_ip": "172.31.82.200", + "registry_port": 5000 + } + } + }, + { + "key": "singularity", + "value": { + "index": 2, + "data": { + "container": "rfaas-container-singularity" + } + } + } + ] } diff --git a/server/executor_manager/executor_process.cpp b/server/executor_manager/executor_process.cpp index afdbf2d..f7b5afa 100644 --- a/server/executor_manager/executor_process.cpp +++ b/server/executor_manager/executor_process.cpp @@ -1,4 +1,5 @@ +#include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include "executor_process.hpp" #include "settings.hpp" @@ -81,14 +83,14 @@ namespace rfaas::executor_manager { executor_pin_threads = std::to_string(0);//counter++); else executor_pin_threads = std::to_string(exec.pin_threads); - bool use_docker = exec.use_docker; + auto sandbox_type = exec.sandbox_type; std::string mgr_port = std::to_string(conn.port); std::string mgr_secret = std::to_string(conn.secret); std::string mgr_buf_addr = std::to_string(conn.r_addr); std::string mgr_buf_rkey = std::to_string(conn.r_key); - int mypid = fork(); + int mypid = vfork(); if(mypid < 0) { spdlog::error("Fork failed! {}", mypid); } @@ -96,12 +98,15 @@ namespace rfaas::executor_manager { mypid = getpid(); auto out_file = ("executor_" + std::to_string(mypid)); - spdlog::info("Child fork begins work on PID {}, using Docker? {}", mypid, use_docker); + spdlog::info("Child fork begins work on PID {}, using sandbox {}", mypid, sandbox_serialize(sandbox_type)); int fd = open(out_file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); dup2(fd, 1); dup2(fd, 2); - if(!use_docker) { - const char * argv[] = { + + std::vector argv; + std::vector additional_args; + if(sandbox_type == SandboxType::PROCESS) { + argv = { "executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), @@ -122,53 +127,43 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - int ret = execvp(argv[0], const_cast(&argv[0])); - if(ret == -1) { - spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); - close(fd); - exit(1); - } - } else { - //const char * argv[] = { - // "docker_rdma_sriov", "run", - // "--rm", - // "--net=mynet", "-i", //"-it", - // // FIXME: make configurable - // "--ip=148.187.105.220", - // // FIXME: make configurable - // "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // // FIXME: make configurable - // "rdma-test", - // "/opt/bin/executor", - // "-a", client_addr.c_str(), - // "-p", client_port.c_str(), - // "--polling-mgr", "thread", - // "-r", executor_repetitions.c_str(), - // "-x", executor_recv_buf.c_str(), - // "-s", client_in_size.c_str(), - // "--pin-threads", "true", - // "--fast", client_cores.c_str(), - // "--warmup-iters", executor_warmups.c_str(), - // "--max-inline-data", executor_max_inline.c_str(), - // "--func-size", client_func_size.c_str(), - // "--timeout", client_timeout.c_str(), - // "--mgr-address", conn.addr.c_str(), - // "--mgr-port", mgr_port.c_str(), - // "--mgr-secret", mgr_secret.c_str(), - // "--mgr-buf-addr", mgr_buf_addr.c_str(), - // "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - // nullptr - //}; - const char * argv[] = { - "docker_rdma_sriov", "run", - "--rm", - "--net=mynet", "-i", //"-it", - // FIXME: make configurable - "--ip=148.187.105.250", - // FIXME: make configurable - "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // FIXME: make configurable - "rdma-test", + } else if (sandbox_type == SandboxType::SARUS) { + argv = { + "sarus", "run" + }; + + exec.sandbox_config->generate_args(argv, exec.sandbox_user); + argv.emplace_back(exec.sandbox_name); + argv.emplace_back(exec.sandbox_config->get_executor_path()); + + additional_args = { + "-a", client_addr.c_str(), + "-p", client_port.c_str(), + "--polling-mgr", "thread", + "-r", executor_repetitions.c_str(), + "-x", executor_recv_buf.c_str(), + "-s", client_in_size.c_str(), + "--pin-threads", executor_pin_threads.c_str(), + "--fast", client_cores.c_str(), + "--warmup-iters", executor_warmups.c_str(), + "--max-inline-data", executor_max_inline.c_str(), + "--func-size", client_func_size.c_str(), + "--timeout", client_timeout.c_str(), + "--mgr-address", conn.addr.c_str(), + "--mgr-port", mgr_port.c_str(), + "--mgr-secret", mgr_secret.c_str(), + "--mgr-buf-addr", mgr_buf_addr.c_str(), + "--mgr-buf-rkey", mgr_buf_rkey.c_str(), + nullptr + }; + } else if (sandbox_type == SandboxType::DOCKER) { + argv = { + "docker_rdma_sriov", "run", "--rm", "-i" + }; + DockerConfiguration config = std::get(*exec.sandbox_config); + config.generate_args(argv); + + additional_args = { "/opt/bin/executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), @@ -189,14 +184,32 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - int ret = execvp(argv[0], const_cast(&argv[0])); - if(ret == -1) { - spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); - close(fd); - exit(1); - } + } else if(sandbox_type == SandboxType::SINGULARITY) { + // Handle Singularity case + SingularityConfiguration config = std::get(*exec.sandbox_config); + } + std::vector cstrings_argv; + std::transform(argv.begin(), argv.end(), std::back_inserter(cstrings_argv), + [](const std::string & input) -> const char* { + return input.c_str(); + } + ); + std::copy(additional_args.begin(), additional_args.end(), std::back_inserter(cstrings_argv)); + + SPDLOG_DEBUG("Executor launch arguments"); + for(const char* str : cstrings_argv) + if(str) + std::cout<(&cstrings_argv.data()[0])); + if(ret == -1) { + spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); + close(fd); + exit(1); } + //close(fd); exit(0); } diff --git a/server/executor_manager/settings.cpp b/server/executor_manager/settings.cpp index a5fcc14..b87364b 100644 --- a/server/executor_manager/settings.cpp +++ b/server/executor_manager/settings.cpp @@ -1,16 +1,44 @@ +#include +#include + #include +#include + #include "settings.hpp" namespace rfaas::executor_manager { + SandboxType sandbox_deserialize(std::string type) + { + static std::map sandboxes = { + {"process", SandboxType::PROCESS}, + {"docker", SandboxType::DOCKER}, + {"sarus", SandboxType::SARUS}, + {"singularity", SandboxType::SINGULARITY} + }; + return sandboxes.at(type); + } + + std::string sandbox_serialize(SandboxType type) + { + static std::map sandboxes = { + {SandboxType::PROCESS, "process"}, + {SandboxType::DOCKER, "docker"}, + {SandboxType::SARUS, "sarus"}, + {SandboxType::SINGULARITY, "singularity"} + }; + return sandboxes.at(type); + } + Settings Settings::deserialize(std::istream & in) { Settings settings{}; cereal::JSONInputArchive archive_in(in); archive_in(cereal::make_nvp("config", settings)); archive_in(cereal::make_nvp("executor", settings.exec)); + archive_in(cereal::make_nvp("sandbox-configuration", settings.sandboxes)); // read RDMA device details rfaas::device_data * dev = rfaas::devices::instance().device(settings.rdma_device); @@ -24,8 +52,69 @@ namespace rfaas::executor_manager { settings.exec.max_inline_data = dev->max_inline_data; settings.exec.recv_buffer_size = dev->default_receive_buffer_size; + if (settings.exec.sandbox_type != SandboxType::PROCESS) { + settings.exec.sandbox_config = &settings.sandboxes.at(settings.exec.sandbox_type); + } + // FIXME: should be sent with request + if (settings.exec.sandbox_type == SandboxType::SARUS) { + SarusConfiguration config = std::get(*settings.exec.sandbox_config); + settings.exec.sandbox_user = config.user; + settings.exec.sandbox_name = config.name; + for(auto & mount : config.mounts) + std::cerr << mount << std::endl; + } + return settings; } + void SarusConfiguration::generate_args(std::vector & args, const std::string & user) const + { + for(auto & dev : this->devices) + args.emplace_back(rdmalib::impl::string_format("--device=%s", dev)); + + for(auto & mount : this->mount_filesystem) { + std::string user_partition{mount}; + user_partition = std::regex_replace(user_partition, std::regex{R"(\{user\})"}, user); + args.emplace_back(rdmalib::impl::string_format("--mount=type=bind,source=%s,destination=%s", user_partition, user_partition)); + } + + for(auto & mount : this->mounts) + args.emplace_back(rdmalib::impl::string_format("--mount=type=bind,source=%s,destination=%s", mount, mount)); + + for(auto & [key, value] : this->env) { + args.emplace_back("-e"); + args.emplace_back(rdmalib::impl::string_format("%s=%s", key, value)); + } + + } + + std::string SarusConfiguration::get_executor_path() const + { + // Horrible hack - we need to get the location of the executor. + // We assume that rFaaS is built on the shared filesystem that is mounted + // in the container. + // Furtheermore, since rFaaS is built in a single directory, we know + // that executor should be located in the same directory as + // executor_manager. + + // This works only on Linux! + auto path = std::filesystem::canonical("/proc/self/exe").parent_path(); + return path / "executor"; + } + + void DockerConfiguration::generate_args(std::vector & args) const { + std::string ip_arg = "--ip=" + ip; + std::string volume_arg = volume + ":/opt"; + std::string net_arg = "--net=" + network; + std::string registry_port = std::to_string(this->registry_port); + std::string docker_image = registry_ip + ":" + registry_port + "/" + image; + + args.emplace_back(net_arg); + args.emplace_back(ip_arg); + args.emplace_back("--volume"); + args.emplace_back(volume_arg); + args.emplace_back(docker_image); + } } + diff --git a/server/executor_manager/settings.hpp b/server/executor_manager/settings.hpp index 5347c07..5f0a61e 100644 --- a/server/executor_manager/settings.hpp +++ b/server/executor_manager/settings.hpp @@ -1,19 +1,109 @@ - #ifndef __RFAAS_EXECUTOR_MANAGER_SETTINGS_HPP__ #define __RFAAS_EXECUTOR_MANAGER_SETTINGS_HPP__ +#include #include #include #include +#include +#include +#include +#include + +namespace rfaas::executor_manager { + + enum class SandboxType { + PROCESS = 0, + DOCKER = 1, + SARUS = 2, + SINGULARITY = 3, + }; + + SandboxType sandbox_deserialize(std::string type); + + std::string sandbox_serialize(SandboxType type); + +} namespace rfaas::executor_manager { + struct DockerConfiguration { + std::string image; + std::string network; + std::string ip; + std::string volume; + std::string registry_ip; + int registry_port; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(image), CEREAL_NVP(network), + CEREAL_NVP(ip), CEREAL_NVP(volume), + CEREAL_NVP(registry_ip), CEREAL_NVP(registry_port) + ); + } + + void generate_args(std::vector & args) const; + }; + + struct SarusConfiguration { + std::string user; + std::string name; + std::vector devices; + std::vector mounts; + std::vector mount_filesystem; + std::map env; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(user), CEREAL_NVP(name), + CEREAL_NVP(devices), CEREAL_NVP(mounts), + CEREAL_NVP(mount_filesystem), CEREAL_NVP(env) + ); + } + + void generate_args(std::vector & args, const std::string & user) const; + + /** + * In the Sarus container, we cannot build the executor since we need to + * compile with Cray headers. + * Thus, we have to mount the rFaaS build directory and access executor + * this way. + **/ + std::string get_executor_path() const; + + }; + + struct SingularityConfiguration { + // TODO: Add other singularity options here + std::string container; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(container) + ); + } + }; + + using SandboxConfiguration = std::variant; + struct ExecutorSettings { - bool use_docker; + SandboxType sandbox_type; + SandboxConfiguration* sandbox_config; + std::string sandbox_user; + std::string sandbox_name; + int repetitions; int warmup_iters; int recv_buffer_size; @@ -23,10 +113,12 @@ namespace rfaas::executor_manager { template void load(Archive & ar ) { + std::string sandbox_type; ar( - CEREAL_NVP(use_docker), CEREAL_NVP(repetitions), + CEREAL_NVP(sandbox_type), CEREAL_NVP(repetitions), CEREAL_NVP(warmup_iters), CEREAL_NVP(pin_threads) ); + this->sandbox_type = sandbox_deserialize(sandbox_type); } }; @@ -46,6 +138,8 @@ namespace rfaas::executor_manager { // Passed to the scheduled executor ExecutorSettings exec; + std::map sandboxes; + template void load(Archive & ar ) { @@ -61,4 +155,22 @@ namespace rfaas::executor_manager { } +namespace cereal +{ + + template inline + std::string save_minimal(Archive const &, rfaas::executor_manager::SandboxType const & t) + { + return rfaas::executor_manager::sandbox_serialize(t); + } + + template inline + void load_minimal( Archive const &, rfaas::executor_manager::SandboxType & t, std::string const & value) + { + t = rfaas::executor_manager::sandbox_deserialize(value); + } + +} + #endif + From 4bc28b2e2153c64d2f5d548bbb80e2759e0e972b Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 31 May 2023 20:37:25 +0000 Subject: [PATCH 2/8] Finished changes. Testing now --- .gitignore | 5 +++++ rdmalib/include/rdmalib/util.hpp | 21 ++++++++++++++++++++ server/executor_manager/executor_process.cpp | 19 ++++++++++++------ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 8364c9e..b39f2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,8 @@ bin/ configuration/ volumes/ containers/config/htpasswd + +benchmarks/warm_benchmarker +benchmarks/parallel_invocations +benchmarks/cold_benchmarker +benchmarks/cpp_interface diff --git a/rdmalib/include/rdmalib/util.hpp b/rdmalib/include/rdmalib/util.hpp index c1dd811..14df5b1 100644 --- a/rdmalib/include/rdmalib/util.hpp +++ b/rdmalib/include/rdmalib/util.hpp @@ -2,6 +2,8 @@ #ifndef __RDMALIB_UTIL_HPP__ #define __RDMALIB_UTIL_HPP__ +#include +#include #include namespace rdmalib { namespace impl { @@ -73,6 +75,25 @@ namespace rdmalib { namespace impl { assert(ptr); } + template + const char* to_cstr(const StrType & str) + { + return str.c_str(); + } + + // Code borrowed from StackOverflow https://stackoverflow.com/questions/2342162/stdstring-formatting-like-sprintf + template + std::string string_format( const std::string& format, Args... args ) + { + int size_s = std::snprintf( nullptr, 0, format.c_str(), to_cstr(args)... ) + 1; + if(size_s <= 0) + return ""; + auto size = static_cast(size_s); + std::unique_ptr buf(new char[size]); + std::snprintf(buf.get(), size, format.c_str(), to_cstr(args)...); + return std::string(buf.get(), buf.get() + size - 1); + } + }} #endif diff --git a/server/executor_manager/executor_process.cpp b/server/executor_manager/executor_process.cpp index f7b5afa..105b892 100644 --- a/server/executor_manager/executor_process.cpp +++ b/server/executor_manager/executor_process.cpp @@ -83,7 +83,7 @@ namespace rfaas::executor_manager { executor_pin_threads = std::to_string(0);//counter++); else executor_pin_threads = std::to_string(exec.pin_threads); - auto sandbox_type = exec.sandbox_type; + auto sandbox_type = exec.sandbox_type; std::string mgr_port = std::to_string(conn.port); std::string mgr_secret = std::to_string(conn.secret); @@ -127,16 +127,21 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - } else if (sandbox_type == SandboxType::SARUS) { + } else if(sandbox_type == SandboxType::SARUS) { + argv = { "sarus", "run" }; - exec.sandbox_config->generate_args(argv, exec.sandbox_user); + SarusConfiguration config = std::get(*exec.sandbox_config); + config.generate_args(argv, exec.sandbox_user); + argv.emplace_back(exec.sandbox_name); - argv.emplace_back(exec.sandbox_config->get_executor_path()); + + argv.emplace_back(config.get_executor_path()); additional_args = { + "/opt/bin/executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), "--polling-mgr", "thread", @@ -156,8 +161,9 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - } else if (sandbox_type == SandboxType::DOCKER) { - argv = { + + } else if(sandbox_type == SandboxType::DOCKER) { + argv = { "docker_rdma_sriov", "run", "--rm", "-i" }; DockerConfiguration config = std::get(*exec.sandbox_config); @@ -184,6 +190,7 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; + } else if(sandbox_type == SandboxType::SINGULARITY) { // Handle Singularity case SingularityConfiguration config = std::get(*exec.sandbox_config); From cc1ef722fb04c42389cd6163fcdf0f6c098ccf71 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 31 May 2023 20:40:56 +0000 Subject: [PATCH 3/8] Added Docker scripts --- scripts/docker/build_base.sh | 3 ++ scripts/docker/init_docker.sh | 80 ++++++++++++++++++++++++++++ scripts/docker/push_image.sh | 13 +++++ scripts/docker/registry.yaml | 20 +++++++ scripts/docker/rfaas-base.Dockerfile | 14 +++++ scripts/docker/run_registry.sh | 47 ++++++++++++++++ 6 files changed, 177 insertions(+) create mode 100755 scripts/docker/build_base.sh create mode 100755 scripts/docker/init_docker.sh create mode 100755 scripts/docker/push_image.sh create mode 100644 scripts/docker/registry.yaml create mode 100644 scripts/docker/rfaas-base.Dockerfile create mode 100755 scripts/docker/run_registry.sh diff --git a/scripts/docker/build_base.sh b/scripts/docker/build_base.sh new file mode 100755 index 0000000..2b2657a --- /dev/null +++ b/scripts/docker/build_base.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t rfaas-base:latest - < rfaas-base.Dockerfile diff --git a/scripts/docker/init_docker.sh b/scripts/docker/init_docker.sh new file mode 100755 index 0000000..c976dae --- /dev/null +++ b/scripts/docker/init_docker.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# This script builds the rfaas-base docker container, pushes it to the +# registry, sets up a docker volume, and generates example configuration +# json which should then be updated in config/executor_manager.json. +# The script also configures a docker network for suitable use with +# docker_rdma_sriov + +# NOTE: Run this script from repo root, and make sure the sriov docker plugin +# is installed + +set -e + +if [ $# -lt 2 ]; then + echo "usage: ./init_docker.sh [ ]" + exit +fi + +REG_IP=$1 # IP or name of the docker registry +REG_PORT=$2 # Port of the docker registry +NET_MODE=$3 # Docker networking mode -- sriov or host +DEVICE=$4 # The RDMA adapter to use for networking +SUBNET=$5 # Subnet for the docker network + +IMG_NAME=rfaas-base +REG_IMG=$REG_IP:$REG_PORT/$IMG_NAME + +# Build the docker container, login and push to the registry +docker build -t $IMG_NAME - < containers/rfaas-base.Dockerfile +echo "built rfaas-base image" +docker login $REG_IP:$REG_PORT +echo "logged into docker daemon" + +if docker push $REG_IMG; then + echo "ERROR: make sure a docker registry is actually running on $REG_IP:$REG_PORT. + Start one with scripts/run_registry.sh" + exit +else + echo "pushed rfaas-base image to $REG_IMG" +fi + +# Set up docker network +net_name=testnet +if ["$NET_MODE" = "sriov"]; then + docker network create -d sriov --subnet=$SUBNET -o netdevice=$DEVICE $net_name +elif ["$NET_MODE" = "host"]; then + net_name="host" +else + echo "ERROR: invalid networking mode $NET_MODE. Valid options are sriov and host." + exit +fi +echo "set up docker network" + +# Configure volume +volume=$(pwd)/volumes/rfaas-test/opt # Do not put a trailing slash +mkdir -p $volume/bin +cp bin/executor $volume/bin +cp examples/libfunctions.so $volume + +# Print json to be updated +config=$(jq -n --arg use_docker "true" \ + --arg image "$REG_IMG" \ + --arg network "$net_name" \ + --arg ip "" \ + --arg volume $volume \ + --arg registry_ip "$REG_IP" \ + --arg registry_port "$REG_PORT" \ + '{ + "image": $image, + "network": $network, + "ip": $ip, + "volume": $volume, + "registry_ip": $registry_ip, + "registry_port": $registry_port + }' +) + +echo "Update config/executor_manager.json with" +echo "$config" + diff --git a/scripts/docker/push_image.sh b/scripts/docker/push_image.sh new file mode 100755 index 0000000..40beeb3 --- /dev/null +++ b/scripts/docker/push_image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Push an image to the local docker registry + +if [ $# -lt 3 ]; then + echo "usage: ./push_image.sh "; + exit +fi + +IMAGE=$1 +IP=$2 +PORT=$3 +docker push $IP:$PORT/$IMAGE diff --git a/scripts/docker/registry.yaml b/scripts/docker/registry.yaml new file mode 100644 index 0000000..3ed7a10 --- /dev/null +++ b/scripts/docker/registry.yaml @@ -0,0 +1,20 @@ +version: "3" + +services: + registry: + image: registry:2 + container_name: rfaas-registry + ports: + - "5000:5000" + environment: + REGISTRY_AUTH: htpasswd + REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd + REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm + REGISTRY_STORAGE_DELETE_ENABLED: "true" + #REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt + #REGISTRY_HTTP_TLS_KEY: /certs/domain.unencrypted.key + #REGISTRY_HTTP_SECRET: supersecrettext + volumes: + - /home/ubuntu/rfaas/containers/registry:/var/lib/registry + - /home/ubuntu/rfaas/containers/config:/auth + #- /home/ubuntu/rfaas/containers/config/certs:/certs diff --git a/scripts/docker/rfaas-base.Dockerfile b/scripts/docker/rfaas-base.Dockerfile new file mode 100644 index 0000000..401297e --- /dev/null +++ b/scripts/docker/rfaas-base.Dockerfile @@ -0,0 +1,14 @@ +# Container version should be same ubuntu version as where `executor` +# was built (due to glibc versioning) + +FROM ubuntu:22.04 + +#ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && apt-get upgrade -y \ + && apt-get install -y \ + libibverbs-dev librdmacm-dev + +RUN mkdir -p /opt/bin +WORKDIR "/opt/bin" + diff --git a/scripts/docker/run_registry.sh b/scripts/docker/run_registry.sh new file mode 100755 index 0000000..d3cb0eb --- /dev/null +++ b/scripts/docker/run_registry.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Run this script on the server where you want the docker registry to be hosted +# Recommended to host the registry on the same server as the executor manager + +# NOTE: Run this script from the repo root + +PORT=5000 +NAME="rfaas-registry" + +set -e + +# Make password file if it doesn't already exist +cfg=containers/config +mkdir -p $cfg +if [ -s $cfg/htpasswd ]; then + echo "htpasswd exists" +else + sudo htpasswd -Bc $cfg/htpasswd $USER + echo "created htpasswd file" +fi + +# Generate certs to use TLS (if they dont already exist) +## if [ -s $cfg/certs/domain.key ]; then +## echo "using certs in $cfg/certs" +## else +## mkdir -p $cfg/certs +## openssl genpkey -algorithm RSA -out $cfg/certs/domain.key -aes256 +## +## openssl req -new \ +## -key $cfg/certs/domain.key \ +## -out $cfg/certs/domain.csr \ +## -addext 'subjectAltName = IP:172.31.82.200' +## +## openssl x509 -req -days 365 \ +## -in $cfg/certs/domain.csr \ +## -signkey $cfg/certs/domain.key \ +## -out $cfg/certs/domain.crt +## +## openssl rsa -in $cfg/certs/domain.key -out $cfg/certs/domain.unencrypted.key +## echo "generated certs in $cfg/certs" +## fi + +# Start registry +sudo docker-compose -f scripts/registry.yaml up -d +echo "started docker registry $NAME on port $PORT" + From 1227a6356a4cb2d8480cdd7b967e56e0b8f62b32 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 31 May 2023 20:52:48 +0000 Subject: [PATCH 4/8] Added help to CLIs --- server/executor/opts.cpp | 6 ++++++ server/executor_manager/opts.cpp | 6 ++++++ server/resource_manager/opts.cpp | 3 +-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/server/executor/opts.cpp b/server/executor/opts.cpp index 5e5adc8..77ca53e 100644 --- a/server/executor/opts.cpp +++ b/server/executor/opts.cpp @@ -30,9 +30,15 @@ namespace server { ("mgr-secret", "Use selected port", cxxopts::value()) ("mgr-buf-addr", "Use selected port", cxxopts::value()) ("mgr-buf-rkey", "Use selected port", cxxopts::value()) + ("h,help", "Print usage") ; auto parsed_options = options.parse(argc, argv); + if (parsed_options.count("help")) { + std::cout << options.help() << std::endl; + exit(0); + } + Options result; result.address = parsed_options["address"].as(); result.port = parsed_options["port"].as(); diff --git a/server/executor_manager/opts.cpp b/server/executor_manager/opts.cpp index c1712d0..e9e24eb 100644 --- a/server/executor_manager/opts.cpp +++ b/server/executor_manager/opts.cpp @@ -13,9 +13,15 @@ namespace rfaas::executor_manager { ("device-database", "JSON configuration of devices.", cxxopts::value()) ("skip-resource-manager", "Ignore resource manager and don't connect to it.", cxxopts::value()->default_value("false")) ("v,verbose", "Verbose output", cxxopts::value()->default_value("false")) + ("h,help", "Print usage") ; auto parsed_options = options.parse(argc, argv); + if (parsed_options.count("help")) { + std::cout << options.help() << std::endl; + exit(0); + } + Options result; result.json_config = parsed_options["config"].as(); result.device_database = parsed_options["device-database"].as(); diff --git a/server/resource_manager/opts.cpp b/server/resource_manager/opts.cpp index 7a10ee0..4102308 100644 --- a/server/resource_manager/opts.cpp +++ b/server/resource_manager/opts.cpp @@ -22,8 +22,7 @@ namespace rfaas::resource_manager { ("h,help", "Print usage", cxxopts::value()->default_value("false")) ; auto parsed_options = options.parse(argc, argv); - if(parsed_options.count("help")) - { + if (parsed_options.count("help")) { std::cout << options.help() << std::endl; exit(0); } From d7075805adefa7d018a9328f2fed731811c6d86c Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Fri, 30 Jun 2023 18:34:29 +0000 Subject: [PATCH 5/8] Made registry mount configurable. Began writing documentation --- docs/docker.md | 32 ++++++++++++++++++++++++++++++++ scripts/docker/registry.yaml | 4 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 docs/docker.md diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..66066eb --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,32 @@ +# Running with Docker + +To use the Docker execution environment or to run a private Docker registry, some configuration is necessary. + +## Using the Docker Execution Environment +rFaaS supports running functions within a Docker container. The Docker image in `scripts/docker/rfaas-base.Dockerfile` is capable of running the functions provided by `example/`, and any other functions that don't require extra dependencies. This Dockerfile serves as a good base to execute your own custom functions. + +To use rFaaS with Docker, consult the example configuration in `config/executor_manager.json`. Set the `sandbox_type` parameter to `"docker"`. Then, configure the Docker-specific parameters under `"key": "docker"`. Here is an example: + +```json +{ + "key": "docker", + "value": { + "index": 0, + "data": { + "image": "rfaas-registry/rfaas-base", + "network": "mynet", + "ip": "172.31.82.202", + "volume": "/path/to/rfaas/bin/", + "registry_ip": "172.31.82.200", + "registry_port": 5000 + } + } +} +``` + +Note that the `"volume"` field must point to a directory containing the `executor` binary built by rFaaS. + +## Local Registry +A script to handle starting a private Docker registry is provided in `scripts/docker/run_registry.sh`. Running a Docker registry requires a mount point, which +ust be configured by the `REGISTRY_LOCATION` parameter in `scripts/docker/registry.yaml`. You can place this mount anywhere on the host machine. + diff --git a/scripts/docker/registry.yaml b/scripts/docker/registry.yaml index 3ed7a10..62d213e 100644 --- a/scripts/docker/registry.yaml +++ b/scripts/docker/registry.yaml @@ -15,6 +15,6 @@ services: #REGISTRY_HTTP_TLS_KEY: /certs/domain.unencrypted.key #REGISTRY_HTTP_SECRET: supersecrettext volumes: - - /home/ubuntu/rfaas/containers/registry:/var/lib/registry - - /home/ubuntu/rfaas/containers/config:/auth + - REGISTRY_LOCATION:/var/lib/registry + - REGISTRY_LOCATION:/auth #- /home/ubuntu/rfaas/containers/config/certs:/certs From 4e34cc40f32e22341aa98326bb1d3c2c4547b177 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Sat, 1 Jul 2023 01:39:32 +0000 Subject: [PATCH 6/8] [exec mgr] Fix a bug regarding execvp nullptrs execvp expects a nullptr at the end as a char*. The previous method of doing this resulted in a 'std::logic_error' what(): basic_string::_M_construct null not valid error, since we were trying to construct a std::string with a nullptr. This commit fixes this bug. --- server/executor_manager/executor_process.cpp | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/server/executor_manager/executor_process.cpp b/server/executor_manager/executor_process.cpp index 105b892..b5c372d 100644 --- a/server/executor_manager/executor_process.cpp +++ b/server/executor_manager/executor_process.cpp @@ -99,6 +99,7 @@ namespace rfaas::executor_manager { auto out_file = ("executor_" + std::to_string(mypid)); spdlog::info("Child fork begins work on PID {}, using sandbox {}", mypid, sandbox_serialize(sandbox_type)); + int fd = open(out_file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); dup2(fd, 1); dup2(fd, 2); @@ -124,8 +125,7 @@ namespace rfaas::executor_manager { "--mgr-port", mgr_port.c_str(), "--mgr-secret", mgr_secret.c_str(), "--mgr-buf-addr", mgr_buf_addr.c_str(), - "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - nullptr + "--mgr-buf-rkey", mgr_buf_rkey.c_str() }; } else if(sandbox_type == SandboxType::SARUS) { @@ -159,7 +159,6 @@ namespace rfaas::executor_manager { "--mgr-secret", mgr_secret.c_str(), "--mgr-buf-addr", mgr_buf_addr.c_str(), "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - nullptr }; } else if(sandbox_type == SandboxType::DOCKER) { @@ -188,7 +187,6 @@ namespace rfaas::executor_manager { "--mgr-secret", mgr_secret.c_str(), "--mgr-buf-addr", mgr_buf_addr.c_str(), "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - nullptr }; } else if(sandbox_type == SandboxType::SINGULARITY) { @@ -196,6 +194,7 @@ namespace rfaas::executor_manager { SingularityConfiguration config = std::get(*exec.sandbox_config); } + // Concat argv and additional_args std::vector cstrings_argv; std::transform(argv.begin(), argv.end(), std::back_inserter(cstrings_argv), [](const std::string & input) -> const char* { @@ -204,12 +203,18 @@ namespace rfaas::executor_manager { ); std::copy(additional_args.begin(), additional_args.end(), std::back_inserter(cstrings_argv)); + // Add a nullptr to the end of the arguments list (as expected by execvp) + cstrings_argv.push_back(nullptr); + + // Print arguments for debug SPDLOG_DEBUG("Executor launch arguments"); - for(const char* str : cstrings_argv) - if(str) - std::cout<(&cstrings_argv.data()[0])); if(ret == -1) { spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); From 11e687db31c13e23b6cb79eb861288101b6dfc74 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 4 Oct 2023 00:43:46 +0200 Subject: [PATCH 7/8] Remove gitignore; we'll merge and clear it later --- .gitignore | 54 ------------------------------------------------------ 1 file changed, 54 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b39f2b9..0000000 --- a/.gitignore +++ /dev/null @@ -1,54 +0,0 @@ -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# CMake -CMakeLists.txt.user -CMakeCache.txt -CMakeFiles -CMakeScripts -Testing -Makefile -cmake_install.cmake -install_manifest.txt -compile_commands.json -CTestTestfile.cmake -_deps -bin/ -configuration/ -volumes/ -containers/config/htpasswd - -benchmarks/warm_benchmarker -benchmarks/parallel_invocations -benchmarks/cold_benchmarker -benchmarks/cpp_interface From b0f1dfc8c76922d669475a88136301711a3451dd Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 4 Oct 2023 00:57:07 +0200 Subject: [PATCH 8/8] [rdmalib] Remove incorrect include --- rdmalib/include/rdmalib/util.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/rdmalib/include/rdmalib/util.hpp b/rdmalib/include/rdmalib/util.hpp index 14df5b1..784fb0d 100644 --- a/rdmalib/include/rdmalib/util.hpp +++ b/rdmalib/include/rdmalib/util.hpp @@ -3,7 +3,6 @@ #define __RDMALIB_UTIL_HPP__ #include -#include #include namespace rdmalib { namespace impl {