From 7a8a06e9d6b1372e0814ff2738d587a6602fcb71 Mon Sep 17 00:00:00 2001 From: Andreas Peters Date: Wed, 29 May 2024 22:18:56 +0200 Subject: [PATCH 1/3] ADD: cni support for the docker executor. --- src/docker/executor.cpp | 311 +++++++++++++++++- src/docker/executor.hpp | 26 +- src/launcher/docker_executor.cpp | 12 +- src/slave/containerizer/docker.cpp | 2 + src/tests/CMakeLists.txt | 1 + src/tests/containerizer/docker_cni_tests.cpp | 326 +++++++++++++++++++ 6 files changed, 672 insertions(+), 6 deletions(-) create mode 100644 src/tests/containerizer/docker_cni_tests.cpp diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp index 8ab165b1155..529ba71fe5f 100644 --- a/src/docker/executor.cpp +++ b/src/docker/executor.cpp @@ -15,6 +15,7 @@ // limitations under the License. #include +#include #include #include @@ -22,6 +23,7 @@ #include #include +#include #include #include #include @@ -37,6 +39,7 @@ #include #include #include +#include #include #include #ifdef __WINDOWS__ @@ -66,6 +69,9 @@ #include "slave/constants.hpp" +namespace io = process::io; +namespace spec = mesos::internal::slave::cni::spec; + using namespace mesos; using namespace process; @@ -101,7 +107,9 @@ class DockerExecutorProcess : public ProtobufProcess const string& launcherDir, const map& taskEnvironment, const Option& defaultContainerDNS, - bool cgroupsEnableCfs) + bool cgroupsEnableCfs, + const string& network_cni_plugins_dir, + const string& network_cni_config_dir) : ProcessBase(ID::generate("docker-executor")), killed(false), terminated(false), @@ -116,6 +124,8 @@ class DockerExecutorProcess : public ProtobufProcess taskEnvironment(taskEnvironment), defaultContainerDNS(defaultContainerDNS), cgroupsEnableCfs(cgroupsEnableCfs), + network_cni_plugins_dir(network_cni_plugins_dir), + network_cni_config_dir(network_cni_config_dir), stop(Nothing()), inspect(Nothing()) {} @@ -285,6 +295,7 @@ class DockerExecutorProcess : public ProtobufProcess inspectLoop.then(defer(self(), [=](const Docker::Container& container) { if (!killed) { containerPid = container.pid; + containerId = container.id; // TODO(alexr): Use `protobuf::createTaskStatus()` // instead of manually setting fields. @@ -318,7 +329,26 @@ class DockerExecutorProcess : public ProtobufProcess } }; - if (container.ipAddress.isSome()) { + bool useCNI = false; +#ifdef __linux__ + if (network_cni_config_dir.size() > 0 && network_cni_plugins_dir.size() > 0) { + Try> cni = attachCNI(task); + if (!cni.isError()) { + Subprocess cniSub = std::get<0>(cni.get()); + std::string cniNetworkName = std::get<1>(cni.get()); + + WIFEXITED(cniSub.status().get().get()); + Try cniIP = attachCNISuccess(cniSub); + if (cniIP.isError()) { + LOG(ERROR) << cniIP.error(); + } else { + useCNI = true; + setIPAddresses(cniIP.get(), false); + } + } + } +#endif + if (container.ipAddress.isSome() && !useCNI) { setIPAddresses(container.ipAddress.get(), false); } @@ -593,6 +623,11 @@ class DockerExecutorProcess : public ProtobufProcess if (!killed) { killed = true; +#ifdef __linux__ + // detach container network + detachCNI(); +#endif + // Send TASK_KILLING if task is not killed by completion timeout and // the framework can handle it. if (!killedByTaskCompletionTimeout && @@ -751,6 +786,11 @@ class DockerExecutorProcess : public ProtobufProcess message = "Container " + WSTRINGIFY(status); } +#ifdef __linux__ + // detach container network + detachCNI(); +#endif + LOG(INFO) << message; CHECK_SOME(taskId); @@ -785,6 +825,261 @@ class DockerExecutorProcess : public ProtobufProcess driver.get()->stop(); } + // Return the content of the CNI plugin config file and return it as json object + Try getNetworkConfigJSON(const string& fname) + { + Try read = os::read(fname); + if (read.isError()) { + return Error("Cannot read CNI plugin file: " + read.error()); + } + + Try parse = JSON::parse(read.get()); + if (parse.isError()) { + return Error("Cannot parse CNI plugin file." + parse.error()); + } + + return parse; + } + + // Search the config file of the network in the path + Try getNetworkConfigFile(const string& network, const string& path) + { + struct dirent *ent; + DIR *dir; + JSON::Object parse; + + // path can include mutiple path's seperated by ":". + char *sPath = strtok(const_cast(path.c_str()), ":"); + while (sPath != NULL) { + if ((dir = opendir(sPath)) != NULL) { + while ((ent = readdir(dir)) != NULL) { + if (ent->d_type != DT_REG || !strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + + std::string fname = string(sPath) + "/" + string(ent->d_name); + Try read = os::read(fname); + if (read.isError()) { + LOG(ERROR) << "Cannot read CNI network config file: " << fname; + continue; + } + + Try parse = JSON::parse(read.get()); + if (parse.isError()) { + LOG(ERROR) << "Cannot parse CNI network config file."; + continue; + } + + Result name = parse->at("name"); + if (!name.isSome()) { + return Error( + "Cannot determine the 'name' of the CNI network for this " + "configuration " + + (name.isNone() ? "'" : ("': " + name.error()))); + } + + // Verify the configuration is for this network + if (network == name->value) { + return fname; + } + } + closedir(dir); + } + sPath = strtok(NULL, ":"); + } + return Error("Cannot find a CNI plugin config for this network name: " + network); + } + + // attach CNI at the container + Try> attachCNI(const TaskInfo& task) + { + container = task.container(); + return commandCNI(container, "ADD"); + } + + // detach CNI from the container + Try> detachCNI() + { + return commandCNI(container, "DEL"); + } + + // function to attach or detach CNI from/to the container + Try> commandCNI( + const mesos::ContainerInfo& container, + const string command) + { + // Loop over all network objects + int ifIndex = 1; + foreach (const mesos::NetworkInfo& networkInfo, container.network_infos()) { + if (!networkInfo.has_name()) { + continue; + } + + const string networkName = networkInfo.name(); + const string ifName = "eth" + stringify(ifIndex++); + + // Find the right plugin config file of the networkname + Try networkConfigFile = getNetworkConfigFile(networkName, network_cni_config_dir); + if (networkConfigFile.isError()) { + LOG(ERROR) << "Could not find the CNI plugin to use for network " + << networkName + << ". Deactivate CNI."; + continue; + } + + Try networkConfigJSON = getNetworkConfigJSON(networkConfigFile.get()); + if (networkConfigJSON.isError()) { + LOG(ERROR) << "Could not get valid CNI configuration for network '" + << networkName << "': " << networkConfigJSON.error(); + continue; + } + + // Invoke CNI plugin to get the plugin type + Result cniPluginType = networkConfigJSON->at("type"); + if (cniPluginType.isError()) { + LOG(ERROR) << cniPluginType.error(); + continue; + } + + if (!cniPluginType.isSome()) { + LOG(ERROR) << "Could not find CNI plugin type to use for network '" << networkName; + continue; + } + + map environment; + environment["CNI_COMMAND"] = command; + environment["CNI_CONTAINERID"] = stringify(containerId); + environment["CNI_PATH"] = network_cni_plugins_dir; + environment["CNI_NETNS"] = "/proc/" + std::to_string(containerPid.get()) + "/ns/net"; + environment["CNI_IFNAME"] = ifName; + + // set the PATH in case the plugin need to use iptables + Option value = os::getenv("PATH"); + if (value.isSome()) { + environment["PATH"] = value.get(); + } else { + environment["PATH"] = os::host_default_path(); + } + + // Invoke the CNI plugin. + Option plugin = os::which( + cniPluginType->value, + network_cni_plugins_dir); + + if (plugin.isNone()) { + LOG(ERROR) << "Cannot load CNI plugin: " << cniPluginType->value; + continue; + } + + // attach cni plugin + if (command.compare("ADD") == 0) { + LOG(INFO) << "Invoking CNI plugin '" << plugin.get() + << "' to attach container " << containerId + << " to network '" << networkName << "'"; + } else { + // detach cni plugin + LOG(INFO) << "Invoking CNI plugin '" << plugin.get() + << "' to dettach container " << containerId + << " from network '" << networkName << "'"; + } + LOG(INFO) << "CNI_COMMAND: " << environment["CNI_COMMAND"]; + LOG(INFO) << "CNI_CONTAINERID: " << environment["CNI_CONTAINERID"]; + LOG(INFO) << "CNI_NETNS: " << environment["CNI_NETNS"]; + LOG(INFO) << "CNI_PATH: " << environment["CNI_PATH"]; + LOG(INFO) << "CNI_IFNAME: " << environment["CNI_IFNAME"]; + + if (command.compare("ADD") == 0) { + LOG(INFO) << "Using network configuration '" + << stringify(networkConfigJSON.get()) + << "' for container " << containerId; + } + + Try s = subprocess( + plugin.get(), + {plugin.get()}, + Subprocess::PATH(networkConfigFile.get()), + Subprocess::PIPE(), + Subprocess::PIPE(), + nullptr, + environment); + + if (s.isError()) { + LOG(ERROR) << "Cannot invoke CNI plugin: " << s.error(); + continue; + } + + return std::make_tuple(s.get(), networkName); + } + return ::Error("Could not match networkinfo to any CNI"); + } + + Try attachCNISuccess( + const Try sub) + { + + const Future>& subStat = sub->status(); + const Future& output = io::read(sub->out().get()); + const Future& error = io::read(sub->err().get()); + + if (!subStat.isReady()) { + return ::Error("Failed to get the exit status of the CNI plugin subprocess."); + } + + // CNI plugin will print result (in case of success) or error (in + // case of failure) to stdout. + if (!output.isReady()) { + return ::Error("Failed to read stdout from the CNI plugin subprocess."); + } + + if (subStat.get() != 0) { + if (!error.isReady()) { + return ::Error("Failed to read stderr from the CNI plugin subprocess."); + } + + return ::Error("The CNI plugin failed to attach container " + stringify(containerId) + + " to CNI network '" + "': stdout='" + output.get() + "', stderr='" + + error.get() + "'"); + } + + Try network = parseNetworkInfo(output.get()); + if (network.isError()) { + return ::Error("Failed to parse the output of the CNI plugin:" + network.error()); + } + + LOG(INFO) << output.get(); + + return network->ip_addresses(0).ip_address().c_str(); + } + + Try parseNetworkInfo(const string& s) + { + NetworkInfo ni; + Try json = JSON::parse(s); + if (json.isError()) { + return ::Error("JSON parse CNI network info failed: " + json.error()); + } + + Result ip4 = json->at("ip4"); + if (ip4.isError()) { + return ::Error("JSON parse CNI network info failed: " + ip4.error()); + } + + Result ipAddress = ip4->at("ip"); + if (ip4.isError()) { + return ::Error("JSON parse CNI network info failed: " + ipAddress.error()); + } + + // remove netmask from the IP string + std::size_t pos = ipAddress->value.find("/"); + if (pos != std::string::npos) { + ipAddress->value.erase(pos); + } + + ni.add_ip_addresses()->set_ip_address(ipAddress->value); + + return ni; + } + void launchCheck(const TaskInfo& task) { // TODO(alexr): Implement general checks support, see MESOS-7250. @@ -856,6 +1151,10 @@ class DockerExecutorProcess : public ProtobufProcess map taskEnvironment; Option defaultContainerDNS; bool cgroupsEnableCfs; + string network_cni_plugins_dir; + string network_cni_config_dir; + string containerId; + mesos::ContainerInfo container; Option killPolicy; Option>> run; @@ -879,7 +1178,9 @@ DockerExecutor::DockerExecutor( const string& launcherDir, const map& taskEnvironment, const Option& defaultContainerDNS, - bool cgroupsEnableCfs) + bool cgroupsEnableCfs, + const string& network_cni_plugins_dir, + const string& network_cni_config_dir) { process = Owned(new DockerExecutorProcess( docker, @@ -890,7 +1191,9 @@ DockerExecutor::DockerExecutor( launcherDir, taskEnvironment, defaultContainerDNS, - cgroupsEnableCfs)); + cgroupsEnableCfs, + network_cni_plugins_dir, + network_cni_config_dir)); spawn(process.get()); } diff --git a/src/docker/executor.hpp b/src/docker/executor.hpp index 768c2e1af13..af88836b152 100644 --- a/src/docker/executor.hpp +++ b/src/docker/executor.hpp @@ -90,6 +90,20 @@ struct Flags : public virtual mesos::internal::logging::Flags "Cgroups feature flag to enable hard limits on CPU resources\n" "via the CFS bandwidth limiting subfeature.\n", false); + + add(&Flags::network_cni_plugins_dir, + "network_cni_plugins_dir", + "A search path for CNI plugin binaries. The docker executer\n" + "will find CNI plugins under these set of directories so that\n" + "it can execute the plugins to add/delete container from the CNI\n" + "networks."); + + add(&Flags::network_cni_config_dir, + "network_cni_config_dir", + "Directory path of the CNI network configuration files. For each\n" + "network that containers launched in Mesos agent can connect to,\n" + "the operator should install a network configuration file in JSON\n" + "format in the specified directory."); } Option container; @@ -100,6 +114,8 @@ struct Flags : public virtual mesos::internal::logging::Flags Option launcher_dir; Option task_environment; Option default_container_dns; + Option network_cni_plugins_dir; + Option network_cni_config_dir; bool cgroups_enable_cfs; @@ -123,7 +139,9 @@ class DockerExecutor : public Executor const std::string& launcherDir, const std::map& taskEnvironment, const Option& defaultContainerDNS, - bool cgroupsEnableCfs); + bool cgroupsEnableCfs, + const std::string& network_cni_plugins_dir, + const std::string& network_cni_config_dir); ~DockerExecutor() override; @@ -158,6 +176,12 @@ class DockerExecutor : public Executor private: process::Owned process; + + void attachCNI(const TaskInfo& task); + void attachCNISuccess(const Try sub); + + Try getNetworkConfigJSON(const std::string& fname, NetworkInfo* netInfo); + Try getNetworkConfigFile(const std::string& network, const std::string& path, NetworkInfo* netInfo); }; } // namespace docker { diff --git a/src/launcher/docker_executor.cpp b/src/launcher/docker_executor.cpp index 3f12c78dfed..a092af6e3ea 100644 --- a/src/launcher/docker_executor.cpp +++ b/src/launcher/docker_executor.cpp @@ -148,6 +148,14 @@ int main(int argc, char** argv) << flags.usage("Missing required option --mapped_directory"); } + if (flags.network_cni_plugins_dir.isNone()) { + flags.network_cni_plugins_dir = "/usr/lib/cni/"; + } + + if (flags.network_cni_config_dir.isNone()) { + flags.network_cni_config_dir = "/etc/mesos/cni/net.d"; + } + map taskEnvironment; if (flags.task_environment.isSome()) { // Parse the string as JSON. @@ -245,7 +253,9 @@ int main(int argc, char** argv) flags.launcher_dir.get(), taskEnvironment, defaultContainerDNS, - flags.cgroups_enable_cfs)); + flags.cgroups_enable_cfs, + flags.network_cni_plugins_dir.get(), + flags.network_cni_config_dir.get())); Owned driver( new mesos::MesosExecutorDriver(executor.get())); diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp index 4e4354f81a4..174d0215332 100644 --- a/src/slave/containerizer/docker.cpp +++ b/src/slave/containerizer/docker.cpp @@ -255,6 +255,8 @@ ::mesos::internal::docker::Flags dockerFlags( dockerFlags.mapped_directory = flags.sandbox_directory; dockerFlags.docker_socket = flags.docker_socket; dockerFlags.launcher_dir = flags.launcher_dir; + dockerFlags.network_cni_plugins_dir = flags.network_cni_plugins_dir; + dockerFlags.network_cni_config_dir = flags.network_cni_config_dir; if (taskEnvironment.isSome()) { dockerFlags.task_environment = string(jsonify(taskEnvironment.get())); diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 79f42bd892d..6e6584fe101 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -167,6 +167,7 @@ list(APPEND MESOS_TESTS_SRC containerizer/containerizer_tests.cpp containerizer/cpu_isolator_tests.cpp containerizer/docker_containerizer_tests.cpp + containerizer/docker_cni_tests.cpp containerizer/docker_tests.cpp containerizer/memory_isolator_tests.cpp) diff --git a/src/tests/containerizer/docker_cni_tests.cpp b/src/tests/containerizer/docker_cni_tests.cpp new file mode 100644 index 00000000000..41d524986de --- /dev/null +++ b/src/tests/containerizer/docker_cni_tests.cpp @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY K:5IND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include +#include + +#include + +#include "slave/containerizer/docker.hpp" +#include "slave/containerizer/fetcher.hpp" +#include "slave/containerizer/mesos/isolators/network/cni/cni.hpp" + +#include "slave/paths.hpp" +#include "slave/slave.hpp" +#include "slave/state.hpp" + +#include "tests/flags.hpp" +#include "tests/mesos.hpp" +#include "tests/environment.hpp" +#include "tests/containerizer/docker_common.hpp" +#include "tests/mock_docker.hpp" + + +using namespace process; + +using namespace mesos::internal::slave::paths; +using namespace mesos::internal::slave::state; + +using mesos::internal::master::Master; +using mesos::internal::slave::DockerContainerizer; +using mesos::internal::slave::DockerContainerizerProcess; +using mesos::internal::slave::Fetcher; +using mesos::internal::slave::Slave; + +using mesos::master::detector::MasterDetector; + +using mesos::slave::ContainerConfig; +using mesos::slave::ContainerLogger; +using mesos::slave::ContainerTermination; + +using std::list; +using std::string; +using std::vector; + +using testing::_; +using testing::DoAll; +using testing::DoDefault; +using testing::Eq; +using testing::Invoke; +using testing::Return; + +namespace mesos { +namespace internal { +namespace tests { + +constexpr char MESOS_MOCK_CNI_CONFIG[] = "mockConfig"; + + +static ContainerInfo createDockerInfo(const string& imageName) +{ + ContainerInfo containerInfo; + + containerInfo.set_type(ContainerInfo::DOCKER); + containerInfo.mutable_docker()->set_image(imageName); + + return containerInfo; +} + +class DockerCniTest : public MesosTest +{ +public: + string cniPluginDir; + string cniConfigDir; + + void SetUp() override + { + MesosTest::SetUp(); + + cniPluginDir = path::join(sandbox.get(), "plugins"); + cniConfigDir = path::join(sandbox.get(), "configs"); + + // This is a veth CNI plugin that is written in bash. It creates a + // veth virtual network pair, one end of the pair will be moved to + // container network namespace. + // + // The veth CNI plugin uses 203.0.113.0/24 subnet, it is reserved + // for documentation and examples [rfc5737]. The plugin can + // allocate up to 128 veth pairs. + Try result = setupMockPlugin( + strings::format(R"~( + #!/bin/sh + set -e + IP_ADDR="203.0.113.10/32" + + NETNS="mesos-test-veth-${PPID}" + mkdir -p /var/run/netns + cleanup() { + rm -f /var/run/netns/$NETNS + } + trap cleanup EXIT + ln -sf $CNI_NETNS /var/run/netns/$NETNS + + VETH0="vmesos1" + VETH1="eth0" + + case $CNI_COMMAND in + "ADD"*) + ip link delete $VETH0 2>/dev/null || true + ip netns exec $NETNS ip link delete $VETH1 2>/dev/null || true + + ip link add name $VETH0 type veth peer name $VETH1 > /tmp/cni.log || continue + ip addr add "${IP_ADDR}" dev $VETH0 >> /tmp/cni.log + ip link set $VETH0 up + ip link set $VETH1 netns $NETNS + ip netns exec $NETNS ip addr add "${IP_ADDR}" dev $VETH1 + ip netns exec $NETNS ip link set $VETH1 up + + echo "{" + echo " \"ip4\": {" + echo " \"ip\": \"${IP_ADDR}\"" + echo " }," + echo " \"dns\": {" + echo " \"nameservers\": [ \"8.8.8.8\" ]" + echo " }" + echo "}" + ;; + "DEL"*) + # $VETH0 on host network namespace will be removed automatically. + # If the plugin can't destroy the veth pair, it will be destroyed + # with the container network namespace. + ip link delete $VETH0 2>/dev/null || true + ip netns exec $NETNS ip link del $VETH1 || true + ;; + esac + )~").get()); + + ASSERT_SOME(result); + + // Generate the mock CNI config for veth CNI plugin. + ASSERT_SOME(os::mkdir(cniConfigDir)); + + result = os::write( + path::join(cniConfigDir, MESOS_MOCK_CNI_CONFIG), + R"~( + { + "name": "__MESOS_TEST__", + "type": "mockPlugin" + })~"); + } + + // Generate the mock CNI plugin based on the given script. + Try setupMockPlugin(const string& pluginScript) + { + return setupPlugin(pluginScript, "mockPlugin"); + } + + // Generate the CNI plugin based on the given script. + Try setupPlugin(const string& pluginScript, const string& pluginName) + { + Try mkdir = os::mkdir(cniPluginDir); + if (mkdir.isError()) { + return Error("Failed to mkdir '" + cniPluginDir + "': " + mkdir.error()); + } + + string pluginPath = path::join(cniPluginDir, pluginName); + + Try write = os::write(pluginPath, pluginScript); + if (write.isError()) { + return Error("Failed to write '" + pluginPath + "': " + write.error()); + } + + // Make sure the plugin has execution permission. + Try chmod = os::chmod( + pluginPath, + S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + + if (chmod.isError()) { + return Error("Failed to chmod '" + pluginPath + "': " + chmod.error()); + } + + return Nothing(); + } + + static string containerName(const ContainerID& containerId) + { + return slave::DOCKER_NAME_PREFIX + containerId.value(); + } + +protected: + slave::Flags CreateSlaveFlags() override + { + slave::Flags flags = MesosTest::CreateSlaveFlags(); + + flags.network_cni_plugins_dir = cniPluginDir; + flags.network_cni_config_dir = cniConfigDir; + + + return flags; + } +}; + +// This test tests the functionality of the docker's interfaces. +TEST_F(DockerCniTest, ROOT_DOCKER_ip_interface) +{ + SetUp(); + + Try> master = StartMaster(); + ASSERT_SOME(master); + + MockDocker* mockDocker = + new MockDocker(tests::flags.docker, tests::flags.docker_socket); + + Shared docker(mockDocker); + + slave::Flags flags = CreateSlaveFlags(); + + Fetcher fetcher(flags); + + Try logger = + ContainerLogger::create(flags.container_logger); + + ASSERT_SOME(logger); + + MockDockerContainerizer dockerContainerizer( + flags, + &fetcher, + Owned(logger.get()), + docker); + + Owned detector = master.get()->createDetector(); + + Try> slave = + StartSlave(detector.get(), &dockerContainerizer, flags); + ASSERT_SOME(slave); + + MockScheduler sched; + MesosSchedulerDriver driver( + &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); + + Future frameworkId; + EXPECT_CALL(sched, registered(&driver, _, _)) + .WillOnce(FutureArg<1>(&frameworkId)); + + Future> offers; + EXPECT_CALL(sched, resourceOffers(&driver, _)) + .WillOnce(FutureArg<1>(&offers)) + .WillRepeatedly(Return()); // Ignore subsequent offers. + + driver.start(); + + AWAIT_READY(frameworkId); + + AWAIT_READY(offers); + ASSERT_FALSE(offers->empty()); + + TaskInfo task = createTask( + offers->front().slave_id(), + offers->front().resources(), + "sleep 10; ping 203.0.113.10 -c 1 -q"); + + ContainerInfo containerInfo = createDockerInfo("docker.io/alpine:latest"); + + containerInfo.mutable_docker()->set_network( + ContainerInfo::DockerInfo::BRIDGE); + + task.mutable_container()->CopyFrom(containerInfo); + task.mutable_container()->add_network_infos()->set_name("__MESOS_TEST__"); + + Future containerId; + Future containerConfig; + EXPECT_CALL(dockerContainerizer, launch(_, _, _, _)) + .WillOnce(DoAll(FutureArg<0>(&containerId), + FutureArg<1>(&containerConfig), + Invoke(&dockerContainerizer, + &MockDockerContainerizer::_launch))); + + Future statusStarting; + Future statusRunning; + Future statusFinished; + EXPECT_CALL(sched, statusUpdate(&driver, _)) + .WillOnce(FutureArg<1>(&statusStarting)) + .WillOnce(FutureArg<1>(&statusRunning)) + .WillOnce(FutureArg<1>(&statusFinished)) + .WillRepeatedly(DoDefault()); + + driver.launchTasks(offers.get()[0].id(), {task}); + + AWAIT_READY_FOR(containerId, Seconds(60)); + AWAIT_READY_FOR(statusStarting, Seconds(60)); + EXPECT_EQ(TASK_STARTING, statusStarting->state()); + AWAIT_READY_FOR(statusRunning, Seconds(60)); + EXPECT_EQ(TASK_RUNNING, statusRunning->state()); + ASSERT_TRUE(statusRunning->has_data()); + AWAIT_READY_FOR(statusFinished, Seconds(60)); + EXPECT_EQ(TASK_FINISHED, statusFinished->state()); + + Future> termination = + dockerContainerizer.wait(containerId.get()); + + driver.stop(); + driver.join(); + + AWAIT_READY(termination); + EXPECT_SOME(termination.get()); +} + +} // namespace tests +} // namespace internal +} // namespace mesos From dabf7912ac31f11b8a9ea1e86b5b635eea3a0017 Mon Sep 17 00:00:00 2001 From: Andreas Peters Date: Tue, 5 Nov 2024 09:50:24 +0100 Subject: [PATCH 2/3] FIX: indent --- src/docker/executor.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/docker/executor.hpp b/src/docker/executor.hpp index af88836b152..0b99b821550 100644 --- a/src/docker/executor.hpp +++ b/src/docker/executor.hpp @@ -99,11 +99,11 @@ struct Flags : public virtual mesos::internal::logging::Flags "networks."); add(&Flags::network_cni_config_dir, - "network_cni_config_dir", - "Directory path of the CNI network configuration files. For each\n" - "network that containers launched in Mesos agent can connect to,\n" - "the operator should install a network configuration file in JSON\n" - "format in the specified directory."); + "network_cni_config_dir", + "Directory path of the CNI network configuration files. For each\n" + "network that containers launched in Mesos agent can connect to,\n" + "the operator should install a network configuration file in JSON\n" + "format in the specified directory."); } Option container; From 2ec7693eef6b08bc32f7b1a6b4cf3ba31282d5cb Mon Sep 17 00:00:00 2001 From: Andreas Peters Date: Wed, 5 Feb 2025 22:40:03 +0100 Subject: [PATCH 3/3] REMOVE: cni default values. --- src/docker/executor.cpp | 2 +- src/launcher/docker_executor.cpp | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp index 529ba71fe5f..e4b7d0100c5 100644 --- a/src/docker/executor.cpp +++ b/src/docker/executor.cpp @@ -331,7 +331,7 @@ class DockerExecutorProcess : public ProtobufProcess bool useCNI = false; #ifdef __linux__ - if (network_cni_config_dir.size() > 0 && network_cni_plugins_dir.size() > 0) { + if (!network_cni_config_dir.empty() && !network_cni_plugins_dir.empty()) { Try> cni = attachCNI(task); if (!cni.isError()) { Subprocess cniSub = std::get<0>(cni.get()); diff --git a/src/launcher/docker_executor.cpp b/src/launcher/docker_executor.cpp index a092af6e3ea..da60be2f090 100644 --- a/src/launcher/docker_executor.cpp +++ b/src/launcher/docker_executor.cpp @@ -148,14 +148,6 @@ int main(int argc, char** argv) << flags.usage("Missing required option --mapped_directory"); } - if (flags.network_cni_plugins_dir.isNone()) { - flags.network_cni_plugins_dir = "/usr/lib/cni/"; - } - - if (flags.network_cni_config_dir.isNone()) { - flags.network_cni_config_dir = "/etc/mesos/cni/net.d"; - } - map taskEnvironment; if (flags.task_environment.isSome()) { // Parse the string as JSON.